jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 01:09:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add python 3.4 grammar checking

Browse Source 
DRY grammar testing
This commit is contained in:
rocky
2016-07-27 18:31:56 -04:00
parent e8ed17967c
commit 04ae94ee9e
5 changed files with 58 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
pytest/test_grammar.py
View File

@@ -1,9 +1,18 @@
import pytest
import pytest, re
from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
from uncompyle6.parser import get_python_parser
from uncompyle6.scanner import get_scanner
def test_grammar():
def check_tokens(tokens, opcode_set):
remain_tokens = set(tokens) - opcode_set
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
assert remain_tokens == set([]), \
"Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())
p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
lhs, rhs, tokens, right_recursive = p.checkSets()
expect_lhs = set(['expr1024', 'pos_arg'])
@@ -21,16 +30,15 @@ def test_grammar():
assert unused_rhs == set(rhs)
assert expect_right_recursive == right_recursive
s = get_scanner(PYTHON_VERSION, IS_PYPY)
if PYTHON_VERSION == 2.7:
opcode_set = set(s.opc.opname).union(set(
ignore_set = set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
assert remain_tokens == set([]), \
"Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())
""".split())
if 2.6 <= PYTHON_VERSION <= 2.7:
opcode_set = set(s.opc.opname).union(ignore_set)
check_tokens(tokens, opcode_set)
elif PYTHON_VERSION == 3.4:
ignore_set.add('LOAD_CLASSNAME')
opcode_set = set(s.opc.opname).union(ignore_set)
check_tokens(tokens, opcode_set)

45
uncompyle6/parser.py
View File

@@ -236,17 +236,12 @@ class PythonParser(GenericASTBuilder):
stmt ::= augassign2
augassign1 ::= expr expr inplace_op designator
augassign1 ::= expr expr inplace_op ROT_THREE STORE_SUBSCR
augassign1 ::= expr expr inplace_op ROT_TWO STORE_SLICE+0
augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1
augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2
augassign1 ::= expr expr inplace_op ROT_FOUR STORE_SLICE+3
augassign2 ::= expr DUP_TOP LOAD_ATTR expr
inplace_op ROT_TWO STORE_ATTR
inplace_op ::= INPLACE_ADD
inplace_op ::= INPLACE_SUBTRACT
inplace_op ::= INPLACE_MULTIPLY
inplace_op ::= INPLACE_DIVIDE
inplace_op ::= INPLACE_TRUE_DIVIDE
inplace_op ::= INPLACE_FLOOR_DIVIDE
inplace_op ::= INPLACE_MODULO
@@ -405,15 +400,10 @@ class PythonParser(GenericASTBuilder):
expr ::= unary_expr
expr ::= call_function
expr ::= unary_not
expr ::= unary_convert
expr ::= binary_subscr
expr ::= binary_subscr2
expr ::= load_attr
expr ::= get_iter
expr ::= slice0
expr ::= slice1
expr ::= slice2
expr ::= slice3
expr ::= buildslice2
expr ::= buildslice3
expr ::= yield
@@ -428,7 +418,6 @@ class PythonParser(GenericASTBuilder):
binary_op ::= BINARY_OR
binary_op ::= BINARY_XOR
binary_op ::= BINARY_SUBTRACT
binary_op ::= BINARY_DIVIDE
binary_op ::= BINARY_TRUE_DIVIDE
binary_op ::= BINARY_FLOOR_DIVIDE
binary_op ::= BINARY_MODULO
@@ -442,21 +431,11 @@ class PythonParser(GenericASTBuilder):
unary_op ::= UNARY_INVERT
unary_not ::= expr UNARY_NOT
unary_convert ::= expr UNARY_CONVERT
binary_subscr ::= expr expr BINARY_SUBSCR
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
load_attr ::= expr LOAD_ATTR
get_iter ::= expr GET_ITER
slice0 ::= expr SLICE+0
slice0 ::= expr DUP_TOP SLICE+0
slice1 ::= expr expr SLICE+1
slice1 ::= expr expr DUP_TOPX_2 SLICE+1
slice2 ::= expr expr SLICE+2
slice2 ::= expr expr DUP_TOPX_2 SLICE+2
slice3 ::= expr expr expr SLICE+3
slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
buildslice3 ::= expr expr expr BUILD_SLICE_3
buildslice2 ::= expr expr BUILD_SLICE_2
@@ -465,12 +444,6 @@ class PythonParser(GenericASTBuilder):
_mklambda ::= load_closure mklambda
_mklambda ::= mklambda
# Note: Python < 2.7 doesn't have *POP* or this. Remove from here?
# FIXME: segregate 2.7+
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
or ::= expr jmp_true expr come_from_opt
and ::= expr jmp_false expr come_from_opt
and2 ::= _jump jmp_false COME_FROM expr COME_FROM
@@ -489,14 +462,6 @@ class PythonParser(GenericASTBuilder):
ret_expr_or_cond ::= ret_cond
ret_expr_or_cond ::= ret_cond_not
# Note: Python < 2.7 doesn't have *POP* or this. Remove from here?
# FIXME: segregate 2.7+
ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
stmt ::= return_lambda
stmt ::= conditional_lambda
@@ -508,15 +473,9 @@ class PythonParser(GenericASTBuilder):
compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
_come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list2 _come_from
@@ -561,10 +520,6 @@ class PythonParser(GenericASTBuilder):
designator ::= STORE_GLOBAL
designator ::= STORE_DEREF
designator ::= expr STORE_ATTR
designator ::= expr STORE_SLICE+0
designator ::= expr expr STORE_SLICE+1
designator ::= expr expr STORE_SLICE+2
designator ::= expr expr expr STORE_SLICE+3
designator ::= store_subscr
store_subscr ::= expr expr STORE_SUBSCR
designator ::= unpack

42
uncompyle6/parsers/parse2.py
View File

@@ -43,6 +43,9 @@ class Python2Parser(PythonParser):
def p_stmt2(self, args):
"""
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
exec_stmt ::= expr exprlist EXEC_STMT
"""
def p_print_to(self, args):
@@ -89,8 +92,6 @@ class Python2Parser(PythonParser):
raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
stmt ::= exec_stmt
exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
exec_stmt ::= expr exprlist EXEC_STMT
stmt ::= assert
stmt ::= assert2
@@ -218,8 +219,13 @@ class Python2Parser(PythonParser):
def p_expr2(self, args):
'''
"""
expr ::= LOAD_LOCALS
expr ::= slice0
expr ::= slice1
expr ::= slice2
expr ::= slice3
expr ::= unary_convert
slice0 ::= expr SLICE+0
slice0 ::= expr DUP_TOP SLICE+0
@@ -229,10 +235,38 @@ class Python2Parser(PythonParser):
slice2 ::= expr expr DUP_TOPX_2 SLICE+2
slice3 ::= expr expr expr SLICE+3
slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
unary_convert ::= expr UNARY_CONVERT
# In Python 3, DUP_TOPX_2 is DUP_TOP_TWO
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
'''
"""
def p_slice2(self, args):
"""
designator ::= expr STORE_SLICE+0
designator ::= expr expr STORE_SLICE+1
designator ::= expr expr STORE_SLICE+2
designator ::= expr expr expr STORE_SLICE+3
augassign1 ::= expr expr inplace_op ROT_TWO STORE_SLICE+0
augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1
augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2
augassign1 ::= expr expr inplace_op ROT_FOUR STORE_SLICE+3
slice0 ::= expr SLICE+0
slice0 ::= expr DUP_TOP SLICE+0
slice1 ::= expr expr SLICE+1
slice1 ::= expr expr DUP_TOPX_2 SLICE+1
slice2 ::= expr expr SLICE+2
slice2 ::= expr expr DUP_TOPX_2 SLICE+2
slice3 ::= expr expr expr SLICE+3
slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
"""
def p_op2(self, args):
"""
inplace_op ::= INPLACE_DIVIDE
binary_op ::= BINARY_DIVIDE
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
"""
def add_custom_rules(self, tokens, customize):
'''

8
uncompyle6/parsers/parse3.py
View File

@@ -90,10 +90,6 @@ class Python3Parser(PythonParser):
raise_stmt2 ::= expr expr RAISE_VARARGS_2
raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
stmt ::= exec_stmt
exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
exec_stmt ::= expr exprlist EXEC_STMT
stmt ::= assert
stmt ::= assert2
stmt ::= ifstmt
@@ -115,10 +111,6 @@ class Python3Parser(PythonParser):
del_stmt ::= DELETE_FAST
del_stmt ::= DELETE_NAME
del_stmt ::= DELETE_GLOBAL
del_stmt ::= expr DELETE_SLICE+0
del_stmt ::= expr expr DELETE_SLICE+1
del_stmt ::= expr expr DELETE_SLICE+2
del_stmt ::= expr expr expr DELETE_SLICE+3
del_stmt ::= delete_subscr
delete_subscr ::= expr expr DELETE_SUBSCR
del_stmt ::= expr DELETE_ATTR

2
uncompyle6/parsers/parse34.py
View File

@@ -57,7 +57,7 @@ if __name__ == '__main__':
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set