jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 01:09:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Start 3.6+ var type annotations and decompyle3 merge...

Browse Source 
Although overall an improvement, some new breakage
has occurred and should be fixed.
This commit is contained in:
rocky
2020-01-01 01:42:00 -05:00
parent faf6ea9630
commit 6de57249ed
7 changed files with 403 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

297
uncompyle6/parsers/parse37.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2019 Rocky Bernstein
# Copyright (c) 2017-2020 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -22,7 +22,6 @@ from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parsers.parse37base import Python37BaseParser
class Python37Parser(Python37BaseParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python37Parser, self).__init__(debug_parser)
self.customized = {}
@@ -328,6 +327,26 @@ class Python37Parser(Python37BaseParser):
attributes ::= LOAD_ATTR+
"""
def p_import37(self, args):
"""
stmt ::= import37
# Where does the POP_TOP really belong?
import37 ::= import POP_TOP
attributes ::= IMPORT_FROM ROT_TWO POP_TOP IMPORT_FROM
attributes ::= attributes ROT_TWO POP_TOP IMPORT_FROM
# The 3.7base scanner adds IMPORT_NAME_ATTR
alias ::= IMPORT_NAME_ATTR IMPORT_FROM store
alias ::= IMPORT_NAME_ATTR attributes store
alias ::= IMPORT_NAME_ATTR store
import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist POP_TOP
expr ::= attribute37
attribute37 ::= expr LOAD_METHOD
"""
def p_list_comprehension(self, args):
"""
expr ::= list_comp
@@ -501,108 +520,12 @@ class Python37Parser(Python37BaseParser):
iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
"""
def p_36misc(self, args):
def p_37async(self, args):
"""
sstmt ::= sstmt RETURN_LAST
# 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
for_block ::= l_stmts_opt come_from_loops JUMP_BACK
come_from_loops ::= COME_FROM_LOOP*
whilestmt ::= setup_loop testexpr l_stmts_opt
JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
whilestmt ::= setup_loop testexpr l_stmts_opt
come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
# 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
# RETURN_VALUE is meant. Specifcally this can happen in
# ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
return ::= ret_expr RETURN_END_IF
return ::= ret_expr RETURN_VALUE COME_FROM
return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM
# A COME_FROM is dropped off because of JUMP-to-JUMP optimization
and ::= expr jmp_false expr
and ::= expr jmp_false expr jmp_false
jf_cf ::= JUMP_FORWARD COME_FROM
cf_jf_else ::= come_froms JUMP_FORWARD ELSE
conditional ::= expr jmp_false expr jf_cf expr COME_FROM
async_for_stmt ::= setup_loop expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY COME_FROM
for_block POP_BLOCK
COME_FROM_LOOP
# Adds a COME_FROM_ASYNC_WITH over 3.5
# FIXME: remove corresponding rule for 3.5?
except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
jb_cfs ::= come_from_opt JUMP_BACK come_froms
ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel
# In 3.6+, A sequence of statements ending in a RETURN can cause
# JUMP_FORWARD END_FINALLY to be omitted from try middle
except_return ::= POP_TOP POP_TOP POP_TOP returns
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_return
# Try middle following a returns
except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY
stmt ::= try_except36
try_except36 ::= SETUP_EXCEPT returns except_handler36
opt_come_from_except
try_except36 ::= SETUP_EXCEPT suite_stmts
try_except36 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler36 come_from_opt
# 3.6 omits END_FINALLY sometimes
except_handler36 ::= COME_FROM_EXCEPT except_stmts
except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
stmt ::= tryfinally36
tryfinally36 ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts
tryfinally36 ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts_opt END_FINALLY
except_suite_finalize ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
stmt ::= tryfinally_return_stmt
tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
COME_FROM_FINALLY
compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
"""
def p_37misc(self, args):
"""
stmt ::= import37
stmt ::= async_for_stmt37
stmt ::= async_for_stmt
stmt ::= async_forelse_stmt
# Where does the POP_TOP really belong?
import37 ::= import POP_TOP
alias ::= IMPORT_NAME_ATTR IMPORT_FROM store
alias ::= IMPORT_NAME_ATTR attributes store
alias ::= IMPORT_NAME_ATTR store
import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME_ATTR importlist POP_TOP
async_for_stmt ::= setup_loop expr
GET_AITER
SETUP_EXCEPT GET_ANEXT LOAD_CONST
@@ -641,19 +564,10 @@ class Python37Parser(Python37BaseParser):
COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP POP_BLOCK
else_suite COME_FROM_LOOP
"""
attributes ::= IMPORT_FROM ROT_TWO POP_TOP IMPORT_FROM
attributes ::= attributes ROT_TWO POP_TOP IMPORT_FROM
attribute37 ::= expr LOAD_METHOD
expr ::= attribute37
# long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a
# JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional
# ELSE added to the except_suite. With better flow control perhaps we can
# sort this out better.
except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE
def p_37chained(self, args):
"""
testtrue ::= compare_chained37
testfalse ::= compare_chained37_false
@@ -694,7 +608,10 @@ class Python37Parser(Python37BaseParser):
compare_chained2a_false_37 ELSE
compare_chained2c_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
compare_chained2a_false_37
"""
def p_37conditionals(self, args):
"""
jf_cfs ::= JUMP_FORWARD _come_froms
ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except
@@ -768,6 +685,28 @@ class Python37Parser(Python37BaseParser):
comp_iter ::= comp_body
"""
def p_expr3(self, args):
"""
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM
# a JUMP_FORWARD to another JUMP_FORWARD can get turned into
# a JUMP_ABSOLUTE with no COME_FROM
conditional ::= expr jmp_false expr jump_absolute_else expr
# if_expr_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,
# and we use that to match on to reconstruct the source more accurately
expr ::= if_expr_true
if_expr_true ::= expr JUMP_FORWARD expr COME_FROM
"""
def p_generator_exp3(self, args):
"""
load_genexpr ::= LOAD_GENEXPR
load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
"""
def p_grammar(self, args):
"""
sstmt ::= stmt
@@ -826,6 +765,7 @@ class Python37Parser(Python37BaseParser):
iflaststmtl ::= testexpr c_stmts JUMP_BACK POP_BLOCK
# These are used to keep parse tree indices the same
jump_forward_else ::= JUMP_FORWARD
jump_forward_else ::= JUMP_FORWARD ELSE
jump_forward_else ::= JUMP_FORWARD COME_FROM
jump_absolute_else ::= JUMP_ABSOLUTE ELSE
@@ -950,6 +890,7 @@ class Python37Parser(Python37BaseParser):
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
jitop_come_from ::= JUMP_IF_TRUE_OR_POP COME_FROM
jifop_come_from ::= JUMP_IF_FALSE_OR_POP COME_FROM
or ::= and jitop_come_from expr COME_FROM
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
or ::= expr JUMP_IF_TRUE expr COME_FROM
@@ -963,11 +904,13 @@ class Python37Parser(Python37BaseParser):
testfalse ::= or jmp_false COME_FROM
or ::= expr jmp_true expr
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr come_from_opt
and ::= expr jifop_come_from expr
and ::= expr JUMP_IF_FALSE expr COME_FROM
pjit_come_from ::= POP_JUMP_IF_TRUE COME_FROM
or ::= expr pjit_come_from expr
## FIXME: Is the below needed or is it covered above??
and ::= expr jmp_false expr COME_FROM
or ::= expr jmp_true expr COME_FROM
@@ -983,6 +926,8 @@ class Python37Parser(Python37BaseParser):
"""
stmt ::= if_expr_lambda
stmt ::= conditional_not_lambda
stmt ::= ifstmtl
if_expr_lambda ::= expr jmp_false expr return_if_lambda
return_stmt_lambda LAMBDA_MARKER
conditional_not_lambda
@@ -997,6 +942,10 @@ class Python37Parser(Python37BaseParser):
stmt ::= whileTruestmt
ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms
_ifstmts_jumpl ::= c_stmts JUMP_BACK
_ifstmts_jumpl ::= _ifstmts_jump
ifstmtl ::= testexpr _ifstmts_jumpl
"""
def p_loop_stmt3(self, args):
@@ -1019,12 +968,20 @@ class Python37Parser(Python37BaseParser):
whilestmt ::= setup_loop testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= setup_loop testexpr returns POP_BLOCK
COME_FROM_LOOP
# We can be missing a COME_FROM_LOOP if the "while" statement is nested inside an if/else
# so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
# This is undoubtedly some sort of JUMP optimization going on.
whilestmt ::= setup_loop testexpr l_stmts_opt JUMP_BACK come_froms
POP_BLOCK
while1elsestmt ::= setup_loop l_stmts JUMP_BACK
else_suitel
@@ -1032,11 +989,12 @@ class Python37Parser(Python37BaseParser):
else_suitel COME_FROM_LOOP
whileTruestmt ::= setup_loop l_stmts_opt JUMP_BACK POP_BLOCK
COME_FROM_LOOP
_come_froms
# FIXME: Python 3.? starts adding branch optimization? Put this starting there.
while1stmt ::= setup_loop l_stmts COME_FROM_LOOP
while1stmt ::= setup_loop l_stmts COME_FROM_LOOP JUMP_BACK POP_BLOCK COME_FROM_LOOP
while1stmt ::= setup_loop l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
while1elsestmt ::= setup_loop l_stmts JUMP_BACK
@@ -1047,26 +1005,105 @@ class Python37Parser(Python37BaseParser):
COME_FROM_LOOP
"""
def p_generator_exp3(self, args):
def p_36misc(self, args):
"""
load_genexpr ::= LOAD_GENEXPR
load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
sstmt ::= sstmt RETURN_LAST
# 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
for_block ::= l_stmts_opt come_from_loops JUMP_BACK
come_from_loops ::= COME_FROM_LOOP*
whilestmt ::= setup_loop testexpr l_stmts_opt
JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
whilestmt ::= setup_loop testexpr l_stmts_opt
come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
# 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
# RETURN_VALUE is meant. Specifcally this can happen in
# ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
return ::= ret_expr RETURN_END_IF
return ::= ret_expr RETURN_VALUE COME_FROM
return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA COME_FROM
# A COME_FROM is dropped off because of JUMP-to-JUMP optimization
and ::= expr jmp_false expr
and ::= expr jmp_false expr jmp_false
jf_cf ::= JUMP_FORWARD COME_FROM
cf_jf_else ::= come_froms JUMP_FORWARD ELSE
conditional ::= expr jmp_false expr jf_cf expr COME_FROM
async_for_stmt ::= setup_loop expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY COME_FROM
for_block POP_BLOCK
COME_FROM_LOOP
# Adds a COME_FROM_ASYNC_WITH over 3.5
# FIXME: remove corresponding rule for 3.5?
except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
jb_cfs ::= come_from_opt JUMP_BACK come_froms
ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel
# In 3.6+, A sequence of statements ending in a RETURN can cause
# JUMP_FORWARD END_FINALLY to be omitted from try middle
except_return ::= POP_TOP POP_TOP POP_TOP returns
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_return
# Try middle following a returns
except_handler36 ::= COME_FROM_EXCEPT except_stmts END_FINALLY
stmt ::= try_except36
try_except36 ::= SETUP_EXCEPT returns except_handler36
opt_come_from_except
try_except36 ::= SETUP_EXCEPT suite_stmts
try_except36 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler36 come_from_opt
# 3.6 omits END_FINALLY sometimes
except_handler36 ::= COME_FROM_EXCEPT except_stmts
except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
stmt ::= tryfinally36
tryfinally36 ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts
tryfinally36 ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts_opt END_FINALLY
except_suite_finalize ::= SETUP_FINALLY returns
COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
stmt ::= tryfinally_return_stmt
tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
COME_FROM_FINALLY
compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
"""
def p_expr3(self, args):
def p_37misc(self, args):
"""
expr ::= conditionalnot
conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM
# long except clauses in a loop can sometimes cause a JUMP_BACK to turn into a
# JUMP_FORWARD to a JUMP_BACK. And when this happens there is an additional
# ELSE added to the except_suite. With better flow control perhaps we can
# sort this out better.
except_suite ::= c_stmts_opt POP_EXCEPT jump_except ELSE
# a JUMP_FORWARD to another JUMP_FORWARD can get turned into
# a JUMP_ABSOLUTE with no COME_FROM
conditional ::= expr jmp_false expr jump_absolute_else expr
# if_expr_true are for conditions which always evaluate true
# There is dead or non-optional remnants of the condition code though,
# and we use that to match on to reconstruct the source more accurately
expr ::= if_expr_true
if_expr_true ::= expr JUMP_FORWARD expr COME_FROM
# FIXME: the below is to work around test_grammar expecting a "call" to be
# on the LHS because it is also somewhere on in a rule.
call ::= expr CALL_METHOD_0
"""
def customize_grammar_rules(self, tokens, customize):