jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 00:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Grammar coverage and pruning

Browse Source
This commit is contained in:
rocky
2017-11-23 05:40:30 -05:00
parent d0644e08d7
commit d8a3c2708e
11 changed files with 67 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
test/Makefile
View File

@@ -114,8 +114,14 @@ grammar-coverage-2.6:
#: Get grammar coverage for Python 2.7 #: Get grammar coverage for Python 2.7
grammar-coverage-2.7: grammar-coverage-2.7:
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7 SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13 SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
#: Get grammar coverage for Python 3.2
grammar-coverage-3.2:
rm $(COVER_DIR)/spark-grammar-32.cover || /bin/true
SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
#: Get grammar coverage for Python 3.3 #: Get grammar coverage for Python 3.3
grammar-coverage-3.3: grammar-coverage-3.3:

BIN
test/bytecode_3.1/10_classdec.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.3/01_delete_deref.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.3/01_triple_compare.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.3/10_classdec.pyc Normal file
View File

Binary file not shown.

4
test/simple_source/bug33/01_triple_compare.py Normal file
View File

@@ -0,0 +1,4 @@
# In Python 3.3+ this uses grammar rule
# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
def _is_valid_netmask(self, netmask):
return 0 <= netmask <= self._max_prefixlen

13
uncompyle6/parser.py
View File

@@ -525,21 +525,14 @@ class PythonParser(GenericASTBuilder):
return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER
return_lambda ::= ret_expr RETURN_VALUE_LAMBDA return_lambda ::= ret_expr RETURN_VALUE_LAMBDA
conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER # Doesn't seemt to be used anymore, but other conditional_lambda's are
# conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER
cmp ::= cmp_list cmp ::= cmp_list
cmp ::= compare cmp ::= compare
compare ::= expr expr COMPARE_OP compare ::= expr expr COMPARE_OP
cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP _come_from
_come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE
COMPARE_OP jmp_false
cmp_list2 _come_from
cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
mapexpr ::= BUILD_MAP kvlist mapexpr ::= BUILD_MAP kvlist
kvlist ::= kvlist kv kvlist ::= kvlist kv

4
uncompyle6/parsers/parse26.py
View File

@@ -243,7 +243,9 @@ class Python26Parser(Python2Parser):
""" """
conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt
and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list1 _come_from
cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list2 _come_from
return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
conditional_lambda ::= expr jmp_false_then expr return_if_lambda conditional_lambda ::= expr jmp_false_then expr return_if_lambda

73
uncompyle6/parsers/parse3.py
View File

@@ -336,12 +336,10 @@ class Python3Parser(PythonParser):
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
cmp_list1 ::= expr DUP_TOP ROT_THREE cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
COMPARE_OP JUMP_IF_FALSE_OR_POP cmp_list1 COME_FROM
cmp_list1 COME_FROM cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list1 ::= expr DUP_TOP ROT_THREE cmp_list2 COME_FROM
COMPARE_OP JUMP_IF_FALSE_OR_POP
cmp_list2 COME_FROM
""" """
def p_stmt3(self, args): def p_stmt3(self, args):
@@ -440,7 +438,10 @@ class Python3Parser(PythonParser):
@staticmethod @staticmethod
def call_fn_name(token): def call_fn_name(token):
"""Customize CALL_FUNCTION to add the number of positional arguments""" """Customize CALL_FUNCTION to add the number of positional arguments"""
return '%s_%i' % (token.kind, token.attr) if token.attr is not None:
return '%s_%i' % (token.kind, token.attr)
else:
return '%s_0' % (token.kind)
def custom_build_class_rule(self, opname, i, token, tokens, customize): def custom_build_class_rule(self, opname, i, token, tokens, customize):
''' '''
@@ -620,8 +621,9 @@ class Python3Parser(PythonParser):
call_function ::= expr CALL_METHOD call_function ::= expr CALL_METHOD
""" """
seen_LOAD_BUILD_CLASS = False seen_LOAD_BUILD_CLASS = False
seen_LOAD_DICTCOMP = False seen_LOAD_DICTCOMP = False
seen_LOAD_LISTCOMP = False seen_LOAD_LISTCOMP = False
seen_LOAD_SETCOMP = False
# Loop over instructions adding custom grammar rules based on # Loop over instructions adding custom grammar rules based on
# a specific instruction seen. # a specific instruction seen.
@@ -634,6 +636,13 @@ class Python3Parser(PythonParser):
assign2_pypy ::= expr expr designator designator assign2_pypy ::= expr expr designator designator
""", nop_func) """, nop_func)
has_get_iter_call_function1 = False
n = len(tokens)
for i, token in enumerate(tokens):
if token == 'GET_ITER' and i < n-2 and self.call_fn_name(tokens[i+1]) == 'CALL_FUNCTION_1':
has_get_iter_call_function1 = True
break
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
opname = token.kind opname = token.kind
opname_base = opname[:opname.rfind('_')] opname_base = opname[:opname.rfind('_')]
@@ -766,17 +775,20 @@ class Python3Parser(PythonParser):
continue continue
elif opname == 'LOAD_DICTCOMP': elif opname == 'LOAD_DICTCOMP':
seen_LOAD_DICTCOMP = True seen_LOAD_DICTCOMP = True
rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr " if has_get_iter_call_function1:
"GET_ITER CALL_FUNCTION_1") rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
self.add_make_function_rule(rule_pat, opname, token.attr, customize) "GET_ITER CALL_FUNCTION_1")
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
elif opname == 'LOAD_LISTCOMP': elif opname == 'LOAD_LISTCOMP':
seen_LOAD_LISTCOMP = True seen_LOAD_LISTCOMP = True
continue continue
elif opname == 'LOAD_SETCOMP': elif opname == 'LOAD_SETCOMP':
seen_LOAD_SETCOMP = True
# Should this be generalized and put under MAKE_FUNCTION? # Should this be generalized and put under MAKE_FUNCTION?
rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr " if has_get_iter_call_function1:
"GET_ITER CALL_FUNCTION_1") rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr "
self.add_make_function_rule(rule_pat, opname, token.attr, customize) "GET_ITER CALL_FUNCTION_1")
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
elif opname == 'LOOKUP_METHOD': elif opname == 'LOOKUP_METHOD':
# A PyPy speciality - DRY with parse2 # A PyPy speciality - DRY with parse2
self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD", self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",
@@ -787,24 +799,27 @@ class Python3Parser(PythonParser):
# Note: this probably doesn't handle kwargs proprerly # Note: this probably doesn't handle kwargs proprerly
args_pos, args_kw, annotate_args = token.attr args_pos, args_kw, annotate_args = token.attr
rule_pat = ("genexpr ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' % rule_pat = ('mklambda ::= %sload_closure LOAD_LAMBDA %%s%s' %
('pos_arg '* args_pos, opname)) ('pos_arg '* args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize) self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
rule_pat = ('setcomp ::= %sload_closure LOAD_SETCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_DICTCOMP: if has_get_iter_call_function1:
self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s ' rule_pat = ("genexpr ::= %sload_closure load_genexpr %%s%s expr "
'expr GET_ITER CALL_FUNCTION_1' % "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
('pos_arg '* args_pos, opname), self.add_make_function_rule(rule_pat, opname, token.attr, customize)
opname, token.attr, customize) if seen_LOAD_LISTCOMP:
rule_pat = ('listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_SETCOMP:
rule_pat = ('setcomp ::= %sload_closure LOAD_SETCOMP %%s%s expr '
'GET_ITER CALL_FUNCTION_1' % ('pos_arg ' * args_pos, opname))
self.add_make_function_rule(rule_pat, opname, token.attr, customize)
if seen_LOAD_DICTCOMP:
self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s '
'expr GET_ITER CALL_FUNCTION_1' %
('pos_arg '* args_pos, opname),
opname, token.attr, customize)
# FIXME: kwarg processing is missing here. # FIXME: kwarg processing is missing here.
# Note order of kwargs and pos args changed between 3.3-3.4 # Note order of kwargs and pos args changed between 3.3-3.4

3
uncompyle6/parsers/parse32.py
View File

@@ -42,6 +42,9 @@ class Python32Parser(Python3Parser):
pass pass
def add_custom_rules(self, tokens, customize): def add_custom_rules(self, tokens, customize):
# self.remove_rules("""
# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
# """)
super(Python32Parser, self).add_custom_rules(tokens, customize) super(Python32Parser, self).add_custom_rules(tokens, customize)
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
opname = token.kind opname = token.kind

2
uncompyle6/parsers/parse33.py
View File

@@ -15,6 +15,8 @@ class Python33Parser(Python32Parser):
expr ::= yield_from expr ::= yield_from
yield_from ::= expr expr YIELD_FROM yield_from ::= expr expr YIELD_FROM
cmp_list2 ::= expr COMPARE_OP RETURN_VALUE
# We do the grammar hackery below for semantics # We do the grammar hackery below for semantics
# actions that want c_stmts_opt at index 1 # actions that want c_stmts_opt at index 1