jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 09:22:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

WIP pypy3.6 handling

Browse Source
This commit is contained in:
rocky
2019-05-11 08:52:38 -04:00
parent f17ebf42a9
commit ebb0342b38
11 changed files with 450 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/Makefile
View File

@@ -333,7 +333,9 @@ pypy-3.2 2.4:
$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify $(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
#: PyPy 5.0.x with Python 3.6 ... #: PyPy 5.0.x with Python 3.6 ...
check-bytecode-pypy3.6: 7.1
7.1: 7.1:
$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify

BIN
test/bytecode_pypy3.6/00_assign.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_pypy3.6_run/00_docstring.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_pypy3.6_run/01_fstring.pyc Normal file
View File

Binary file not shown.

1
test/test_pyenvlib.py
View File

@@ -43,6 +43,7 @@ TEST_VERSIONS = (
"pypy3.5-5.7.1-beta", "pypy3.5-5.7.1-beta",
"pypy3.5-5.9.0", "pypy3.5-5.9.0",
"pypy3.5-6.0.0", "pypy3.5-6.0.0",
"pypy3.6-7.1.0",
"native", "native",
) + tuple(python_versions) ) + tuple(python_versions)

433
uncompyle6/parsers/parse2.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2018 Rocky Bernstein # Copyright (c) 2015-2019 Rocky Bernstein
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com> # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# #
# Copyright (c) 1999 John Aycock # Copyright (c) 1999 John Aycock
@@ -31,10 +31,10 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
class Python2Parser(PythonParser):
class Python2Parser(PythonParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python2Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser) super(Python2Parser, self).__init__(SyntaxTree, "stmts", debug=debug_parser)
self.new_rules = set() self.new_rules = set()
def p_print2(self, args): def p_print2(self, args):
@@ -52,7 +52,7 @@ class Python2Parser(PythonParser):
""" """
def p_print_to(self, args): def p_print_to(self, args):
''' """
stmt ::= print_to stmt ::= print_to
stmt ::= print_to_nl stmt ::= print_to_nl
stmt ::= print_nl_to stmt ::= print_nl_to
@@ -62,10 +62,10 @@ class Python2Parser(PythonParser):
print_to_items ::= print_to_items print_to_item print_to_items ::= print_to_items print_to_item
print_to_items ::= print_to_item print_to_items ::= print_to_item
print_to_item ::= DUP_TOP expr ROT_TWO PRINT_ITEM_TO print_to_item ::= DUP_TOP expr ROT_TWO PRINT_ITEM_TO
''' """
def p_grammar(self, args): def p_grammar(self, args):
''' """
sstmt ::= stmt sstmt ::= stmt
sstmt ::= return RETURN_LAST sstmt ::= return RETURN_LAST
@@ -176,12 +176,12 @@ class Python2Parser(PythonParser):
jmp_abs ::= JUMP_ABSOLUTE jmp_abs ::= JUMP_ABSOLUTE
jmp_abs ::= JUMP_BACK jmp_abs ::= JUMP_BACK
jmp_abs ::= CONTINUE jmp_abs ::= CONTINUE
''' """
def p_generator_exp2(self, args): def p_generator_exp2(self, args):
''' """
generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
''' """
def p_expr2(self, args): def p_expr2(self, args):
""" """
@@ -252,25 +252,41 @@ class Python2Parser(PythonParser):
this. this.
""" """
if 'PyPy' in customize: if "PyPy" in customize:
# PyPy-specific customizations # PyPy-specific customizations
self.addRule(""" self.addRule(
"""
stmt ::= assign3_pypy stmt ::= assign3_pypy
stmt ::= assign2_pypy stmt ::= assign2_pypy
assign3_pypy ::= expr expr expr store store store assign3_pypy ::= expr expr expr store store store
assign2_pypy ::= expr expr store store assign2_pypy ::= expr expr store store
list_comp ::= expr BUILD_LIST_FROM_ARG for_iter store list_iter list_comp ::= expr BUILD_LIST_FROM_ARG for_iter store list_iter
JUMP_BACK JUMP_BACK
""", nop_func) """,
nop_func,
)
# For a rough break out on the first word. This may # For a rough break out on the first word. This may
# include instructions that don't need customization, # include instructions that don't need customization,
# but we'll do a finer check after the rough breakout. # but we'll do a finer check after the rough breakout.
customize_instruction_basenames = frozenset( customize_instruction_basenames = frozenset(
('BUILD', 'CALL', 'CONTINUE', 'DELETE', (
'DUP', 'EXEC', 'GET', 'JUMP', "BUILD",
'LOAD', 'LOOKUP', 'MAKE', 'SETUP', "CALL",
'RAISE', 'UNPACK')) "CONTINUE",
"DELETE",
"DUP",
"EXEC",
"GET",
"JUMP",
"LOAD",
"LOOKUP",
"MAKE",
"SETUP",
"RAISE",
"UNPACK",
)
)
# Opcode names in the custom_seen_ops set have rules that get added # Opcode names in the custom_seen_ops set have rules that get added
# unconditionally and the rules are constant. So they need to be done # unconditionally and the rules are constant. So they need to be done
@@ -284,139 +300,191 @@ class Python2Parser(PythonParser):
# Do a quick breakout before testing potentially # Do a quick breakout before testing potentially
# each of the dozen or so instruction in if elif. # each of the dozen or so instruction in if elif.
if (opname[:opname.find('_')] not in customize_instruction_basenames if (
or opname in custom_seen_ops): opname[: opname.find("_")] not in customize_instruction_basenames
or opname in custom_seen_ops
):
continue continue
opname_base = opname[:opname.rfind('_')] opname_base = opname[: opname.rfind("_")]
# The order of opname listed is roughly sorted below # The order of opname listed is roughly sorted below
if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'): if opname_base in ("BUILD_LIST", "BUILD_SET", "BUILD_TUPLE"):
# We do this complicated test to speed up parsing of # We do this complicated test to speed up parsing of
# pathelogically long literals, especially those over 1024. # pathelogically long literals, especially those over 1024.
build_count = token.attr build_count = token.attr
thousands = (build_count//1024) thousands = build_count // 1024
thirty32s = ((build_count//32) % 32) thirty32s = (build_count // 32) % 32
if thirty32s > 0: if thirty32s > 0:
rule = "expr32 ::=%s" % (' expr' * 32) rule = "expr32 ::=%s" % (" expr" * 32)
self.add_unique_rule(rule, opname_base, build_count, customize) self.add_unique_rule(rule, opname_base, build_count, customize)
if thousands > 0: if thousands > 0:
self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), self.add_unique_rule(
opname_base, build_count, customize) "expr1024 ::=%s" % (" expr32" * 32),
collection = opname_base[opname_base.find('_')+1:].lower() opname_base,
rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + build_count,
'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) customize,
self.add_unique_rules([ )
"expr ::= %s" % collection, collection = opname_base[opname_base.find("_") + 1 :].lower()
rule], customize) rule = (
("%s ::= " % collection)
+ "expr1024 " * thousands
+ "expr32 " * thirty32s
+ "expr " * (build_count % 32)
+ opname
)
self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
continue continue
elif opname_base == 'BUILD_MAP': elif opname_base == "BUILD_MAP":
if opname == 'BUILD_MAP_n': if opname == "BUILD_MAP_n":
# PyPy sometimes has no count. Sigh. # PyPy sometimes has no count. Sigh.
self.add_unique_rules([ self.add_unique_rules(
'kvlist_n ::= kvlist_n kv3', [
'kvlist_n ::=', "kvlist_n ::= kvlist_n kv3",
'dict ::= BUILD_MAP_n kvlist_n', "kvlist_n ::=",
], customize) "dict ::= BUILD_MAP_n kvlist_n",
],
customize,
)
if self.version >= 2.7: if self.version >= 2.7:
self.add_unique_rule( self.add_unique_rule(
'dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store ' "dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store "
'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST', "comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST",
'dict_comp_func', 0, customize) "dict_comp_func",
0,
customize,
)
else: else:
kvlist_n = ' kv3' * token.attr kvlist_n = " kv3" * token.attr
rule = "dict ::= %s%s" % (opname, kvlist_n) rule = "dict ::= %s%s" % (opname, kvlist_n)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
continue continue
elif opname_base == 'BUILD_SLICE': elif opname_base == "BUILD_SLICE":
slice_num = token.attr slice_num = token.attr
if slice_num == 2: if slice_num == 2:
self.add_unique_rules([ self.add_unique_rules(
'expr ::= build_slice2', [
'build_slice2 ::= expr expr BUILD_SLICE_2' "expr ::= build_slice2",
], customize) "build_slice2 ::= expr expr BUILD_SLICE_2",
],
customize,
)
else: else:
assert slice_num == 3, ("BUILD_SLICE value must be 2 or 3; is %s" % assert slice_num == 3, (
slice_num) "BUILD_SLICE value must be 2 or 3; is %s" % slice_num
self.add_unique_rules([ )
'expr ::= build_slice3', self.add_unique_rules(
'build_slice3 ::= expr expr expr BUILD_SLICE_3', [
], customize) "expr ::= build_slice3",
"build_slice3 ::= expr expr expr BUILD_SLICE_3",
],
customize,
)
continue continue
elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', elif opname_base in (
'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): "CALL_FUNCTION",
"CALL_FUNCTION_VAR",
"CALL_FUNCTION_VAR_KW",
"CALL_FUNCTION_KW",
):
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
# number of apply equiv arguments: # number of apply equiv arguments:
nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3 nak = (len(opname_base) - len("CALL_FUNCTION")) // 3
rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ rule = (
+ 'expr ' * nak + opname "call ::= expr "
elif opname_base == 'CALL_METHOD': + "expr " * args_pos
+ "kwarg " * args_kw
+ "expr " * nak
+ opname
)
elif opname_base == "CALL_METHOD":
# PyPy only - DRY with parse3 # PyPy only - DRY with parse3
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
# number of apply equiv arguments: # number of apply equiv arguments:
nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 nak = (len(opname_base) - len("CALL_METHOD")) // 3
rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ rule = (
+ 'expr ' * nak + opname "call ::= expr "
elif opname == 'CONTINUE_LOOP': + "expr " * args_pos
self.addRule('continue ::= CONTINUE_LOOP', nop_func) + "kwarg " * args_kw
+ "expr " * nak
+ opname
)
elif opname == "CONTINUE_LOOP":
self.addRule("continue ::= CONTINUE_LOOP", nop_func)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'DELETE_ATTR': elif opname == "DELETE_ATTR":
self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func) self.addRule("del_stmt ::= expr DELETE_ATTR", nop_func)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname.startswith('DELETE_SLICE'): elif opname.startswith("DELETE_SLICE"):
self.addRule(""" self.addRule(
"""
del_expr ::= expr del_expr ::= expr
del_stmt ::= del_expr DELETE_SLICE+0 del_stmt ::= del_expr DELETE_SLICE+0
del_stmt ::= del_expr del_expr DELETE_SLICE+1 del_stmt ::= del_expr del_expr DELETE_SLICE+1
del_stmt ::= del_expr del_expr DELETE_SLICE+2 del_stmt ::= del_expr del_expr DELETE_SLICE+2
del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3 del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3
""", nop_func) """,
nop_func,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
self.check_reduce['del_expr'] = 'AST' self.check_reduce["del_expr"] = "AST"
continue continue
elif opname == 'DELETE_DEREF': elif opname == "DELETE_DEREF":
self.addRule(""" self.addRule(
"""
stmt ::= del_deref_stmt stmt ::= del_deref_stmt
del_deref_stmt ::= DELETE_DEREF del_deref_stmt ::= DELETE_DEREF
""", nop_func) """,
nop_func,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'DELETE_SUBSCR': elif opname == "DELETE_SUBSCR":
self.addRule(""" self.addRule(
"""
del_stmt ::= delete_subscript del_stmt ::= delete_subscript
delete_subscript ::= expr expr DELETE_SUBSCR delete_subscript ::= expr expr DELETE_SUBSCR
""", nop_func) """,
self.check_reduce['delete_subscript'] = 'AST' nop_func,
)
self.check_reduce["delete_subscript"] = "AST"
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'GET_ITER': elif opname == "GET_ITER":
self.addRule(""" self.addRule(
"""
expr ::= get_iter expr ::= get_iter
attribute ::= expr GET_ITER attribute ::= expr GET_ITER
""", nop_func) """,
nop_func,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'): elif opname_base in ("DUP_TOPX", "RAISE_VARARGS"):
# FIXME: remove these conditions if they are not needed. # FIXME: remove these conditions if they are not needed.
# no longer need to add a rule # no longer need to add a rule
continue continue
elif opname == 'EXEC_STMT': elif opname == "EXEC_STMT":
self.addRule(""" self.addRule(
"""
stmt ::= exec_stmt stmt ::= exec_stmt
exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
exec_stmt ::= expr exprlist EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT
exprlist ::= expr+ exprlist ::= expr+
""", nop_func) """,
nop_func,
)
continue continue
elif opname == 'JUMP_IF_NOT_DEBUG': elif opname == "JUMP_IF_NOT_DEBUG":
self.addRule(""" self.addRule(
"""
jmp_true_false ::= POP_JUMP_IF_TRUE jmp_true_false ::= POP_JUMP_IF_TRUE
jmp_true_false ::= POP_JUMP_IF_FALSE jmp_true_false ::= POP_JUMP_IF_FALSE
stmt ::= assert_pypy stmt ::= assert_pypy
@@ -426,107 +494,152 @@ class Python2Parser(PythonParser):
assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false
LOAD_ASSERT expr CALL_FUNCTION_1 LOAD_ASSERT expr CALL_FUNCTION_1
RAISE_VARARGS_1 COME_FROM RAISE_VARARGS_1 COME_FROM
""", nop_func) """,
nop_func,
)
continue continue
elif opname == 'LOAD_ATTR': elif opname == "LOAD_ATTR":
self.addRule(""" self.addRule(
"""
expr ::= attribute expr ::= attribute
attribute ::= expr LOAD_ATTR attribute ::= expr LOAD_ATTR
""", nop_func) """,
nop_func,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'LOAD_LISTCOMP': elif opname == "LOAD_LISTCOMP":
self.addRule("expr ::= listcomp", nop_func) self.addRule("expr ::= listcomp", nop_func)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'LOAD_SETCOMP': elif opname == "LOAD_SETCOMP":
self.add_unique_rules([ self.add_unique_rules(
"expr ::= set_comp", [
"set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1" "expr ::= set_comp",
], customize) "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1",
],
customize,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'LOOKUP_METHOD': elif opname == "LOOKUP_METHOD":
# A PyPy speciality - DRY with parse3 # A PyPy speciality - DRY with parse3
self.addRule(""" self.addRule(
"""
expr ::= attribute expr ::= attribute
attribute ::= expr LOOKUP_METHOD attribute ::= expr LOOKUP_METHOD
""", """,
nop_func) nop_func,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname_base == 'MAKE_FUNCTION': elif opname_base == "MAKE_FUNCTION":
if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': if i > 0 and tokens[i - 1] == "LOAD_LAMBDA":
self.addRule('mklambda ::= %s LOAD_LAMBDA %s' % self.addRule(
('pos_arg ' * token.attr, opname), nop_func) "mklambda ::= %s LOAD_LAMBDA %s"
rule = 'mkfunc ::= %s LOAD_CODE %s' % ('expr ' * token.attr, opname) % ("pos_arg " * token.attr, opname),
elif opname_base == 'MAKE_CLOSURE': nop_func,
)
rule = "mkfunc ::= %s LOAD_CODE %s" % ("expr " * token.attr, opname)
elif opname_base == "MAKE_CLOSURE":
# FIXME: use add_unique_rules to tidy this up. # FIXME: use add_unique_rules to tidy this up.
if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': if i > 0 and tokens[i - 1] == "LOAD_LAMBDA":
self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' % self.addRule(
('expr ' * token.attr, opname), nop_func) "mklambda ::= %s load_closure LOAD_LAMBDA %s"
% ("expr " * token.attr, opname),
nop_func,
)
if i > 0: if i > 0:
prev_tok = tokens[i-1] prev_tok = tokens[i - 1]
if prev_tok == 'LOAD_GENEXPR': if prev_tok == "LOAD_GENEXPR":
self.add_unique_rules([ self.add_unique_rules(
('generator_exp ::= %s load_closure LOAD_GENEXPR %s expr' [
' GET_ITER CALL_FUNCTION_1' % (
('expr ' * token.attr, opname))], customize) "generator_exp ::= %s load_closure LOAD_GENEXPR %s expr"
" GET_ITER CALL_FUNCTION_1"
% ("expr " * token.attr, opname)
)
],
customize,
)
pass pass
self.add_unique_rules([ self.add_unique_rules(
('mkfunc ::= %s load_closure LOAD_CODE %s' % [
('expr ' * token.attr, opname))], customize) (
"mkfunc ::= %s load_closure LOAD_CODE %s"
% ("expr " * token.attr, opname)
)
],
customize,
)
if self.version >= 2.7: if self.version >= 2.7:
if i > 0: if i > 0:
prev_tok = tokens[i-1] prev_tok = tokens[i - 1]
if prev_tok == 'LOAD_DICTCOMP': if prev_tok == "LOAD_DICTCOMP":
self.add_unique_rules([ self.add_unique_rules(
('dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr' [
' GET_ITER CALL_FUNCTION_1' % (
('expr ' * token.attr, opname))], customize) "dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr"
elif prev_tok == 'LOAD_SETCOMP': " GET_ITER CALL_FUNCTION_1"
self.add_unique_rules([ % ("expr " * token.attr, opname)
"expr ::= set_comp", )
('set_comp ::= %s load_closure LOAD_SETCOMP %s expr' ],
' GET_ITER CALL_FUNCTION_1' % customize,
('expr ' * token.attr, opname)) )
], customize) elif prev_tok == "LOAD_SETCOMP":
self.add_unique_rules(
[
"expr ::= set_comp",
(
"set_comp ::= %s load_closure LOAD_SETCOMP %s expr"
" GET_ITER CALL_FUNCTION_1"
% ("expr " * token.attr, opname)
),
],
customize,
)
pass pass
pass pass
continue continue
elif opname == 'SETUP_EXCEPT': elif opname == "SETUP_EXCEPT":
if 'PyPy' in customize: if "PyPy" in customize:
self.add_unique_rules([ self.add_unique_rules(
"stmt ::= try_except_pypy", [
"try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy", "stmt ::= try_except_pypy",
"except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM" "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy",
], customize) "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM",
],
customize,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == 'SETUP_FINALLY': elif opname == "SETUP_FINALLY":
if 'PyPy' in customize: if "PyPy" in customize:
self.addRule(""" self.addRule(
"""
stmt ::= tryfinallystmt_pypy stmt ::= tryfinallystmt_pypy
tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY
suite_stmts_opt END_FINALLY""", nop_func) suite_stmts_opt END_FINALLY""",
nop_func,
)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): elif opname_base in ("UNPACK_TUPLE", "UNPACK_SEQUENCE"):
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
rule = 'unpack ::= ' + opname + ' store' * token.attr rule = "unpack ::= " + opname + " store" * token.attr
elif opname_base == 'UNPACK_LIST': elif opname_base == "UNPACK_LIST":
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
rule = 'unpack_list ::= ' + opname + ' store' * token.attr rule = "unpack_list ::= " + opname + " store" * token.attr
else: else:
continue continue
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
pass pass
self.check_reduce['raise_stmt1'] = 'tokens' self.check_reduce["raise_stmt1"] = "tokens"
self.check_reduce['aug_assign2'] = 'AST' self.check_reduce["aug_assign2"] = "AST"
self.check_reduce['or'] = 'AST' self.check_reduce["or"] = "AST"
# self.check_reduce['_stmts'] = 'AST' # self.check_reduce['_stmts'] = 'AST'
# Dead code testing... # Dead code testing...
@@ -541,24 +654,30 @@ class Python2Parser(PythonParser):
# Dead code testing... # Dead code testing...
# if lhs == 'while1elsestmt': # if lhs == 'while1elsestmt':
# from trepan.api import debug; debug() # from trepan.api import debug; debug()
if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'): if (
lhs in ("aug_assign1", "aug_assign2")
and ast[0]
and ast[0][0] in ("and", "or")
):
return True return True
elif lhs in ('raise_stmt1',): elif lhs in ("raise_stmt1",):
# We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule # We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule
return (tokens[first] == 'LOAD_ASSERT' and (last >= len(tokens))) return tokens[first] == "LOAD_ASSERT" and (last >= len(tokens))
elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')): elif rule == ("or", ("expr", "jmp_true", "expr", "\\e_come_from_opt")):
expr2 = ast[2] expr2 = ast[2]
return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT' return expr2 == "expr" and expr2[0] == "LOAD_ASSERT"
elif lhs in ('delete_subscript', 'del_expr'): elif lhs in ("delete_subscript", "del_expr"):
op = ast[0][0] op = ast[0][0]
return op.kind in ('and', 'or') return op.kind in ("and", "or")
return False return False
class Python2ParserSingle(Python2Parser, PythonParserSingle): class Python2ParserSingle(Python2Parser, PythonParserSingle):
pass pass
if __name__ == '__main__':
if __name__ == "__main__":
# Check grammar # Check grammar
p = Python2Parser() p = Python2Parser()
p.check_grammar() p.check_grammar()

65
uncompyle6/parsers/parse3.py
View File

@@ -432,7 +432,7 @@ class Python3Parser(PythonParser):
else: else:
return "%s_0" % (token.kind) return "%s_0" % (token.kind)
def custom_build_class_rule(self, opname, i, token, tokens, customize): def custom_build_class_rule(self, opname, i, token, tokens, customize, is_pypy):
""" """
# Should the first rule be somehow folded into the 2nd one? # Should the first rule be somehow folded into the 2nd one?
build_class ::= LOAD_BUILD_CLASS mkfunc build_class ::= LOAD_BUILD_CLASS mkfunc
@@ -483,10 +483,18 @@ class Python3Parser(PythonParser):
call_function = call_fn_tok.kind call_function = call_fn_tok.kind
if call_function.startswith("CALL_FUNCTION_KW"): if call_function.startswith("CALL_FUNCTION_KW"):
self.addRule("classdef ::= build_class_kw store", nop_func) self.addRule("classdef ::= build_class_kw store", nop_func)
rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" % ( if is_pypy:
"expr " * (call_fn_tok.attr - 1), args_pos, args_kw = self.get_pos_kw(call_fn_tok)
call_function, rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %s%s%s" % (
) "expr " * (args_pos - 1),
"kwarg " * (args_kw),
call_function,
)
else:
rule = (
"build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s"
% ("expr " * (call_fn_tok.attr - 1), call_function)
)
else: else:
call_function = self.call_fn_name(call_fn_tok) call_function = self.call_fn_name(call_fn_tok)
rule = "build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" % ( rule = "build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" % (
@@ -496,7 +504,7 @@ class Python3Parser(PythonParser):
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
return return
def custom_classfunc_rule(self, opname, token, customize, next_token): def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
""" """
call ::= expr {expr}^n CALL_FUNCTION_n call ::= expr {expr}^n CALL_FUNCTION_n
call ::= expr {expr}^n CALL_FUNCTION_VAR_n call ::= expr {expr}^n CALL_FUNCTION_VAR_n
@@ -514,18 +522,28 @@ class Python3Parser(PythonParser):
# Yes, this computation based on instruction name is a little bit hoaky. # Yes, this computation based on instruction name is a little bit hoaky.
nak = (len(opname) - len("CALL_FUNCTION")) // 3 nak = (len(opname) - len("CALL_FUNCTION")) // 3
token.kind = self.call_fn_name(token)
uniq_param = args_kw + args_pos uniq_param = args_kw + args_pos
# Note: 3.5+ have subclassed this method; so we don't handle # Note: 3.5+ have subclassed this method; so we don't handle
# 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here. # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.
rule = ( if is_pypy and self.version >= 3.6:
"call ::= expr " if token == "CALL_FUNCTION":
+ ("pos_arg " * args_pos) token.kind = self.call_fn_name(token)
+ ("kwarg " * args_kw) rule = (
+ "expr " * nak "call ::= expr "
+ token.kind + ("pos_arg " * args_pos)
) + ("kwarg " * args_kw)
+ token.kind
)
else:
token.kind = self.call_fn_name(token)
rule = (
"call ::= expr "
+ ("pos_arg " * args_pos)
+ ("kwarg " * args_kw)
+ "expr " * nak
+ token.kind
)
self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule(rule, token.kind, uniq_param, customize)
@@ -821,7 +839,9 @@ class Python3Parser(PythonParser):
""" """
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
self.custom_classfunc_rule(opname, token, customize, tokens[i + 1]) self.custom_classfunc_rule(
opname, token, customize, tokens[i + 1], is_pypy
)
# Note: don't add to custom_ops_processed. # Note: don't add to custom_ops_processed.
elif opname_base == "CALL_METHOD": elif opname_base == "CALL_METHOD":
@@ -880,21 +900,30 @@ class Python3Parser(PythonParser):
self.addRule( self.addRule(
""" """
stmt ::= assert_pypy stmt ::= assert_pypy
stmt ::= assert2_pypy", nop_func) stmt ::= assert_not_pypy
stmt ::= assert2_pypy
stmt ::= assert2_not_pypy
assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true
LOAD_ASSERT RAISE_VARARGS_1 COME_FROM LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
assert_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false
LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true
LOAD_ASSERT expr CALL_FUNCTION_1 LOAD_ASSERT expr CALL_FUNCTION_1
RAISE_VARARGS_1 COME_FROM RAISE_VARARGS_1 COME_FROM
assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true
LOAD_ASSERT expr CALL_FUNCTION_1 LOAD_ASSERT expr CALL_FUNCTION_1
RAISE_VARARGS_1 COME_FROM, RAISE_VARARGS_1 COME_FROM
assert2_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false
LOAD_ASSERT expr CALL_FUNCTION_1
RAISE_VARARGS_1 COME_FROM
""", """,
nop_func, nop_func,
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_BUILD_CLASS": elif opname == "LOAD_BUILD_CLASS":
self.custom_build_class_rule(opname, i, token, tokens, customize) self.custom_build_class_rule(
opname, i, token, tokens, customize, is_pypy
)
# Note: don't add to custom_ops_processed. # Note: don't add to custom_ops_processed.
elif opname == "LOAD_CLASSDEREF": elif opname == "LOAD_CLASSDEREF":
# Python 3.4+ # Python 3.4+

16
uncompyle6/parsers/parse36.py
View File

@@ -316,7 +316,7 @@ class Python36Parser(Python35Parser):
pass pass
return return
def custom_classfunc_rule(self, opname, token, customize, next_token): def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
@@ -338,10 +338,14 @@ class Python36Parser(Python35Parser):
self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize) self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
if opname.startswith('CALL_FUNCTION_KW'): if opname.startswith('CALL_FUNCTION_KW'):
self.addRule("expr ::= call_kw36", nop_func) if is_pypy:
values = 'expr ' * token.attr # PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions
rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
self.add_unique_rule(rule, token.kind, token.attr, customize) else:
self.addRule("expr ::= call_kw36", nop_func)
values = 'expr ' * token.attr
rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
self.add_unique_rule(rule, token.kind, token.attr, customize)
elif opname == 'CALL_FUNCTION_EX_KW': elif opname == 'CALL_FUNCTION_EX_KW':
# Note: this doesn't exist in 3.7 and later # Note: this doesn't exist in 3.7 and later
self.addRule("""expr ::= call_ex_kw4 self.addRule("""expr ::= call_ex_kw4
@@ -406,7 +410,7 @@ class Python36Parser(Python35Parser):
""", nop_func) """, nop_func)
pass pass
else: else:
super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token) super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
def reduce_is_invalid(self, rule, ast, tokens, first, last): def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python36Parser, invalid = super(Python36Parser,

24
uncompyle6/scanners/scanner3.py
View File

@@ -260,7 +260,10 @@ class Scanner3(Scanner):
# There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here
assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n
else: else:
assert_can_follow = inst.opname == "POP_JUMP_IF_TRUE" and i + 1 < n assert_can_follow = (
inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE")
and i + 1 < n
)
if assert_can_follow: if assert_can_follow:
next_inst = self.insts[i + 1] next_inst = self.insts[i + 1]
if ( if (
@@ -270,9 +273,7 @@ class Scanner3(Scanner):
): ):
raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] raise_idx = self.offset2inst_index[self.prev_op[inst.argval]]
raise_inst = self.insts[raise_idx] raise_inst = self.insts[raise_idx]
if raise_inst.opname.startswith( if raise_inst.opname.startswith("RAISE_VARARGS"):
"RAISE_VARARGS"
):
self.load_asserts.add(next_inst.offset) self.load_asserts.add(next_inst.offset)
pass pass
pass pass
@@ -428,11 +429,16 @@ class Scanner3(Scanner):
else: else:
opname = "%s_%d" % (opname, pos_args) opname = "%s_%d" % (opname, pos_args)
elif self.is_pypy and opname == "JUMP_IF_NOT_DEBUG": elif self.is_pypy and opname in ("JUMP_IF_NOT_DEBUG", "CALL_FUNCTION"):
# The value in the dict is in special cases in semantic actions, such if opname == "JUMP_IF_NOT_DEBUG":
# as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put # The value in the dict is in special cases in semantic actions, such
# in arbitrary value 0. # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put
customize[opname] = 0 # in arbitrary value 0.
customize[opname] = 0
elif self.version >= 3.6 and argval > 255:
opname = "CALL_FUNCTION_KW"
pass
elif opname == "UNPACK_EX": elif opname == "UNPACK_EX":
# FIXME: try with scanner and parser by # FIXME: try with scanner and parser by
# changing argval # changing argval

8
uncompyle6/semantics/customize.py
View File

@@ -28,8 +28,12 @@ def customize_for_version(self, is_pypy, version):
# PyPy changes # PyPy changes
####################### #######################
TABLE_DIRECT.update({ TABLE_DIRECT.update({
'assert_pypy': ( '%|assert %c\n' , 1 ), 'assert_pypy': ( '%|assert %c\n' , (1, 'assert_expr') ),
'assert2_pypy': ( '%|assert %c, %c\n' , 1, 4 ), 'assert_not_pypy': ( '%|assert not %c\n' , (1, 'assert_exp') ),
'assert2_not_pypy': ( '%|assert not %c, %c\n' , (1, 'assert_exp'),
(4, 'expr') ),
'assert2_pypy': ( '%|assert %c, %c\n' , (1, 'assert_expr'),
(4, 'expr') ),
'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ), 'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ), 'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ),
'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ), 'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ),

147
uncompyle6/semantics/pysource.py
View File

@@ -1566,6 +1566,15 @@ class SourceWalker(GenericASTTraversal, object):
if node[n].kind.startswith("CALL_FUNCTION_KW"): if node[n].kind.startswith("CALL_FUNCTION_KW"):
# 3.6+ starts doing this # 3.6+ starts doing this
kwargs = node[n - 1].attr kwargs = node[n - 1].attr
if self.is_pypy:
# FIXME: this doesn't handle positional and keyword args
# properly. Need to do something more like that below
# in the non-PYPY 3.6 case.
self.template_engine(('(%[0]{attr}=%c)', 1), node[n-1])
return
else:
kwargs = node[n-1].attr
assert isinstance(kwargs, tuple) assert isinstance(kwargs, tuple)
i = n - (len(kwargs) + 1) i = n - (len(kwargs) + 1)
j = 1 + n - node[n].attr j = 1 + n - node[n].attr
@@ -1750,65 +1759,95 @@ class SourceWalker(GenericASTTraversal, object):
else: else:
kv_node = node[1:] kv_node = node[1:]
else: else:
assert node[-1].kind.startswith("kvlist")
kv_node = node[-1]
first_time = True
for kv in kv_node:
assert kv in ("kv", "kv2", "kv3")
# kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
# kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
# kv3 ::= expr expr STORE_MAP
# FIXME: DRY this and the above
indent = self.indent + " " indent = self.indent + " "
if kv == "kv": line_number = self.line_number
self.write(sep) sep = ''
name = self.traverse(kv[-2], indent="") opname = node[-1].kind
if first_time: if self.is_pypy and self.version >= 3.6:
line_number = self.indent_if_source_nl(line_number, indent) if opname.startswith('BUILD_CONST_KEY_MAP'):
first_time = False keys = node[-2].attr
# FIXME: DRY this and the above
for i in range(len(keys)):
key = keys[i]
value = self.traverse(node[i], indent='')
self.write(sep, key, ': ', value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + " "
line_number = self.line_number
pass
pass
pass pass
line_number = self.line_number else:
self.write(name, ": ") if opname.startswith('kvlist'):
value = self.traverse( list_node = node[0]
kv[1], indent=self.indent + (len(name) + 2) * " " else:
) list_node = node
elif kv == "kv2":
self.write(sep) assert list_node[-1].kind.startswith('BUILD_MAP')
name = self.traverse(kv[1], indent="") for i in range(0, len(list_node)-1, 2):
if first_time: key = self.traverse(list_node[i], indent='')
line_number = self.indent_if_source_nl(line_number, indent) value = self.traverse(list_node[i+1], indent='')
first_time = False self.write(sep, key, ': ', value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + " "
line_number = self.line_number
pass
pass
pass pass
line_number = self.line_number elif opname.startswith('kvlist'):
self.write(name, ": ") kv_node = node[-1]
value = self.traverse( first_time = True
kv[-3], indent=self.indent + (len(name) + 2) * " " for kv in kv_node:
) assert kv in ('kv', 'kv2', 'kv3')
elif kv == "kv3":
self.write(sep) # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
name = self.traverse(kv[-2], indent="") # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
if first_time: # kv3 ::= expr expr STORE_MAP
line_number = self.indent_if_source_nl(line_number, indent)
first_time = False # FIXME: DRY this and the above
if kv == 'kv':
self.write(sep)
name = self.traverse(kv[-2], indent='')
if first_time:
line_number = self.indent_if_source_nl(line_number, indent)
first_time = False
pass
line_number = self.line_number
self.write(name, ': ')
value = self.traverse(kv[1], indent=self.indent+(len(name)+2)*' ')
elif kv == 'kv2':
self.write(sep)
name = self.traverse(kv[1], indent='')
if first_time:
line_number = self.indent_if_source_nl(line_number, indent)
first_time = False
pass
line_number = self.line_number
self.write(name, ': ')
value = self.traverse(kv[-3], indent=self.indent+(len(name)+2)*' ')
elif kv == 'kv3':
self.write(sep)
name = self.traverse(kv[-2], indent='')
if first_time:
line_number = self.indent_if_source_nl(line_number, indent)
first_time = False
pass
line_number = self.line_number
self.write(name, ': ')
line_number = self.line_number
value = self.traverse(kv[0], indent=self.indent+(len(name)+2)*' ')
pass
self.write(value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + " "
line_number = self.line_number
pass
pass pass
line_number = self.line_number
self.write(name, ": ")
line_number = self.line_number
value = self.traverse(
kv[0], indent=self.indent + (len(name) + 2) * " "
)
pass
self.write(value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + " "
line_number = self.line_number
pass
pass pass
pass
if sep.startswith(",\n"): if sep.startswith(",\n"):
self.write(sep[1:]) self.write(sep[1:])
if node[0] != "dict_entry": if node[0] != "dict_entry":