jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 01:09:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-3.3-to-3.5

Browse Source
This commit is contained in:
rocky
2024-03-02 05:07:05 -05:00
parent 3721722764 69c5d463e6
commit ac9c7d1047
13 changed files with 340 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/FUNDING.yml vendored
View File

@@ -6,7 +6,7 @@ open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username liberapay: rocky
issuehunt: # Replace with a single IssueHunt username issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username otechie: # Replace with a single Otechie username
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

2
test/stdlib/3.3-exclude.sh
View File

@@ -10,7 +10,7 @@ SKIP_TESTS=(
# tgt.append(elem) # tgt.append(elem)
[test_itertools.py]=1 [test_itertools.py]=1
[test_buffer.py]=1 # FIXME: Works on c90ff51 [test_buffer.py]=pytest # FIXME: Works on c90ff51
[test_cmath.py]=pytest [test_cmath.py]=pytest
[test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines [test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines

9
test/stdlib/3.8-exclude.sh
View File

@@ -28,12 +28,12 @@ SKIP_TESTS=(
# These and the above may be due to new code generation or tests # These and the above may be due to new code generation or tests
# between 3.8.3 and 3.8.5 ? # between 3.8.3 and 3.8.5 ?
[test_decorators.py]=1 # [test_decorators.py]=1 # parse error
[test_dtrace.py]=1 # [test_dtrace.py]=1 # parse error
[test_exceptions.py]=1 # [test_exceptions.py]=1 # parse error
[test_ftplib.py]=1 # [test_ftplib.py]=1 #
[test_gc.py]=1 # [test_gc.py]=1 # FIXME: return return strip_python_stderr(stderr)
[test_gzip.py]=1 # [test_gzip.py]=1 #
[test_hashlib.py]=1 # [test_hashlib.py]=1 #
[test_iter.py]=1 # [test_iter.py]=1 #
@@ -51,7 +51,6 @@ SKIP_TESTS=(
[test_audioop.py]=1 # test failure [test_audioop.py]=1 # test failure
[test_audit.py]=1 # parse error [test_audit.py]=1 # parse error
[test_base64.py]=1 # parse error
[test_baseexception.py]=1 # [test_baseexception.py]=1 #
[test_bigaddrspace.py]=1 # parse error [test_bigaddrspace.py]=1 # parse error
[test_bigmem.py]=1 # parse error [test_bigmem.py]=1 # parse error

12
uncompyle6/parsers/parse2.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2021 Rocky Bernstein # Copyright (c) 2015-2021, 2024 Rocky Bernstein
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com> # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# #
# Copyright (c) 1999 John Aycock # Copyright (c) 1999 John Aycock
@@ -27,11 +27,12 @@ that a later phase can turn into a sequence of ASCII text.
from __future__ import print_function from __future__ import print_function
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
from uncompyle6.parsers.treenode import SyntaxTree
class Python2Parser(PythonParser): class Python2Parser(PythonParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -405,7 +406,6 @@ class Python2Parser(PythonParser):
"CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_VAR_KW",
"CALL_FUNCTION_KW", "CALL_FUNCTION_KW",
): ):
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
# number of apply equiv arguments: # number of apply equiv arguments:
@@ -526,7 +526,7 @@ class Python2Parser(PythonParser):
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == "LOAD_LISTCOMP": elif opname == "LOAD_LISTCOMP":
self.addRule("expr ::= listcomp", nop_func) self.addRule("expr ::= list_comp", nop_func)
custom_seen_ops.add(opname) custom_seen_ops.add(opname)
continue continue
elif opname == "LOAD_SETCOMP": elif opname == "LOAD_SETCOMP":

12
uncompyle6/parsers/parse3.py
View File

@@ -1085,7 +1085,9 @@ class Python3Parser(PythonParser):
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_LISTCOMP": elif opname == "LOAD_LISTCOMP":
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) self.add_unique_rule(
"expr ::= list_comp", opname, token.attr, customize
)
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_SETCOMP": elif opname == "LOAD_SETCOMP":
# Should this be generalized and put under MAKE_FUNCTION? # Should this be generalized and put under MAKE_FUNCTION?
@@ -1154,7 +1156,7 @@ class Python3Parser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname) % ("pos_arg " * pos_args_count, opname)
) )
@@ -1348,14 +1350,14 @@ class Python3Parser(PythonParser):
# 'exprs' in the rule above into a # 'exprs' in the rule above into a
# tuple. # tuple.
rule_pat = ( rule_pat = (
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s " "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
"expr GET_ITER CALL_FUNCTION_1" % (opname,) "expr GET_ITER CALL_FUNCTION_1" % (opname,)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
) )
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname) % ("expr " * pos_args_count, opname)
) )
@@ -1399,7 +1401,7 @@ class Python3Parser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname) % ("expr " * pos_args_count, opname)
) )

12
uncompyle6/parsers/parse37base.py
View File

@@ -723,7 +723,9 @@ class Python37BaseParser(PythonParser):
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_LISTCOMP": elif opname == "LOAD_LISTCOMP":
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) self.add_unique_rule(
"expr ::= list_comp", opname, token.attr, customize
)
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_NAME": elif opname == "LOAD_NAME":
if ( if (
@@ -802,7 +804,7 @@ class Python37BaseParser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * args_pos, opname) % ("pos_arg " * args_pos, opname)
) )
@@ -900,14 +902,14 @@ class Python37BaseParser(PythonParser):
# 'exprs' in the rule above into a # 'exprs' in the rule above into a
# tuple. # tuple.
rule_pat = ( rule_pat = (
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s " "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
"expr GET_ITER CALL_FUNCTION_1" % (opname,) "expr GET_ITER CALL_FUNCTION_1" % (opname,)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
) )
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
@@ -941,7 +943,7 @@ class Python37BaseParser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1 # and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(

506
uncompyle6/parsers/parse38.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2020, 2022-2023 Rocky Bernstein # Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@@ -17,269 +17,272 @@ spark grammar differences over Python 3.7 for Python 3.8
""" """
from __future__ import print_function from __future__ import print_function
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse37 import Python37Parser from uncompyle6.parsers.parse37 import Python37Parser
from uncompyle6.parsers.reducecheck.pop_return import pop_return_check
class Python38Parser(Python37Parser): class Python38Parser(Python37Parser):
def p_38_stmt(self, args): def p_38_stmt(self, args):
""" """
stmt ::= async_for_stmt38 stmt ::= async_for_stmt38
stmt ::= async_forelse_stmt38 stmt ::= async_forelse_stmt38
stmt ::= call_stmt stmt ::= call_stmt
stmt ::= continue stmt ::= continue
stmt ::= for38 stmt ::= for38
stmt ::= forelselaststmt38 stmt ::= forelselaststmt38
stmt ::= forelselaststmtl38 stmt ::= forelselaststmtl38
stmt ::= forelsestmt38 stmt ::= forelsestmt38
stmt ::= try_elsestmtl38 stmt ::= try_elsestmtl38
stmt ::= try_except38 stmt ::= try_except38
stmt ::= try_except38r stmt ::= try_except38r
stmt ::= try_except38r2 stmt ::= try_except38r2
stmt ::= try_except38r3 stmt ::= try_except38r3
stmt ::= try_except38r4 stmt ::= try_except38r4
stmt ::= try_except_as stmt ::= try_except_as
stmt ::= try_except_ret38 stmt ::= try_except_ret38
stmt ::= tryfinally38astmt stmt ::= tryfinally38astmt
stmt ::= tryfinally38rstmt stmt ::= tryfinally38rstmt
stmt ::= tryfinally38rstmt2 stmt ::= tryfinally38rstmt2
stmt ::= tryfinally38rstmt3 stmt ::= tryfinally38rstmt3
stmt ::= tryfinally38stmt stmt ::= tryfinally38stmt
stmt ::= whileTruestmt38 stmt ::= whileTruestmt38
stmt ::= whilestmt38 stmt ::= whilestmt38
call_stmt ::= call call_stmt ::= call
break ::= POP_BLOCK BREAK_LOOP break ::= POP_BLOCK BREAK_LOOP
break ::= POP_BLOCK POP_TOP BREAK_LOOP break ::= POP_BLOCK POP_TOP BREAK_LOOP
break ::= POP_TOP BREAK_LOOP break ::= POP_TOP BREAK_LOOP
break ::= POP_EXCEPT BREAK_LOOP break ::= POP_EXCEPT BREAK_LOOP
# The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed. # The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed.
# Inside an loop we can have this, which can only appear in side a try/except # Inside an loop we can have this, which can only appear in side a try/except
# And it can also appear at the end of the try except. # And it can also appear at the end of the try except.
continue ::= POP_EXCEPT JUMP_BACK continue ::= POP_EXCEPT JUMP_BACK
# FIXME: this should be restricted to being inside a try block # FIXME: this should be restricted to being inside a try block
stmt ::= except_ret38 stmt ::= except_ret38
stmt ::= except_ret38a stmt ::= except_ret38a
# FIXME: this should be added only when seeing GET_AITER or YIELD_FROM # FIXME: this should be added only when seeing GET_AITER or YIELD_FROM
async_for ::= GET_AITER _come_froms async_for ::= GET_AITER _come_froms
SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
async_for_stmt38 ::= expr async_for async_for_stmt38 ::= expr async_for
store for_block store for_block
COME_FROM_FINALLY COME_FROM_FINALLY
END_ASYNC_FOR END_ASYNC_FOR
genexpr_func_async ::= LOAD_ARG func_async_prefix genexpr_func_async ::= LOAD_ARG func_async_prefix
store comp_iter store comp_iter
JUMP_BACK COME_FROM_FINALLY JUMP_BACK COME_FROM_FINALLY
END_ASYNC_FOR END_ASYNC_FOR
# FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of # FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of
# for / forelse is used. Add come froms and check of add up control-flow detection phase. # for / forelse is used. Add come froms and check of add up control-flow detection phase.
async_forelse_stmt38 ::= expr async_forelse_stmt38 ::= expr
GET_AITER GET_AITER
SETUP_FINALLY SETUP_FINALLY
GET_ANEXT GET_ANEXT
LOAD_CONST LOAD_CONST
YIELD_FROM YIELD_FROM
POP_BLOCK
store for_block
COME_FROM_FINALLY
END_ASYNC_FOR
else_suite
# Seems to be used to discard values before a return in a "for" loop
discard_top ::= ROT_TWO POP_TOP
discard_tops ::= discard_top+
return ::= return_expr
discard_tops RETURN_VALUE
return ::= popb_return
return ::= pop_return
return ::= pop_ex_return
except_stmt ::= pop_ex_return
pop_return ::= POP_TOP return_expr RETURN_VALUE
popb_return ::= return_expr POP_BLOCK RETURN_VALUE
pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
# 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
lastl_stmt ::= ifpoplaststmtl
ifpoplaststmtl ::= testexpr POP_TOP c_stmts_opt
ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms
# Keep indices the same in ifelsestmtl
cf_pt ::= COME_FROM POP_TOP
ifelsestmtl ::= testexpr c_stmts cf_pt else_suite
for38 ::= expr get_iter store for_block JUMP_BACK
for38 ::= expr get_for_iter store for_block JUMP_BACK
for38 ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK
for38 ::= expr get_for_iter store for_block
forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite
forelsestmt38 ::= expr get_for_iter store for_block JUMP_BACK _come_froms
else_suite
forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
returns_in_except ::= _stmts except_return_value
except_return_value ::= POP_BLOCK return
except_return_value ::= expr POP_BLOCK RETURN_VALUE
whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK
POP_BLOCK POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK store for_block
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms COME_FROM_FINALLY
whilestmt38 ::= _come_froms testexpr returns POP_BLOCK END_ASYNC_FOR
whilestmt38 ::= _come_froms testexpr l_stmts JUMP_BACK else_suite
whilestmt38 ::= _come_froms testexpr l_stmts come_froms
# while1elsestmt ::= l_stmts JUMP_BACK # Seems to be used to discard values before a return in a "for" loop
whileTruestmt ::= _come_froms l_stmts JUMP_BACK POP_BLOCK discard_top ::= ROT_TWO POP_TOP
while1stmt ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP discard_tops ::= discard_top+
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE
whileTruestmt38 ::= _come_froms pass JUMP_BACK
for_block ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK return ::= return_expr
discard_tops RETURN_VALUE
except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false return ::= popb_return
POP_TOP POP_TOP POP_TOP return ::= pop_return
POP_EXCEPT return ::= pop_ex_return
except_cond_as ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE except_stmt ::= pop_ex_return
POP_TOP STORE_FAST POP_TOP pop_return ::= POP_TOP return_expr RETURN_VALUE
popb_return ::= return_expr POP_BLOCK RETURN_VALUE
pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK # 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
except_handler38 COME_FROM lastl_stmt ::= ifpoplaststmtl
else_suitel opt_come_from_except ifpoplaststmtl ::= testexpr POP_TOP c_stmts_opt
try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms
except_handler38
try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt # Keep indices the same in ifelsestmtl
except_handler38a cf_pt ::= COME_FROM POP_TOP
ifelsestmtl ::= testexpr c_stmts cf_pt else_suite
# suite_stmts has a return for38 ::= expr get_iter store for_block JUMP_BACK
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts for38 ::= expr get_for_iter store for_block JUMP_BACK
except_handler38b for38 ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK
try_except38r ::= SETUP_FINALLY return_except for38 ::= expr get_for_iter store for_block
except_handler38b
return_except ::= stmts POP_BLOCK return forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite
forelsestmt38 ::= expr get_for_iter store for_block JUMP_BACK _come_froms
else_suite
forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
returns_in_except ::= _stmts except_return_value
except_return_value ::= POP_BLOCK return
except_return_value ::= expr POP_BLOCK RETURN_VALUE
whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK
POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms
whilestmt38 ::= _come_froms testexpr returns POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts JUMP_BACK
whilestmt38 ::= _come_froms testexpr l_stmts come_froms
# while1elsestmt ::= l_stmts JUMP_BACK
whileTruestmt ::= _come_froms l_stmts JUMP_BACK POP_BLOCK
while1stmt ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE
whileTruestmt38 ::= _come_froms pass JUMP_BACK
for_block ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK
except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false
POP_TOP POP_TOP POP_TOP
POP_EXCEPT
except_cond_as ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP STORE_FAST POP_TOP
try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38 COME_FROM
else_suitel opt_come_from_except
try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38
try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
except_handler38a
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
try_except38r ::= SETUP_FINALLY return_except
except_handler38b
return_except ::= stmts POP_BLOCK return
# In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
# is a final return in the "except" block. # is a final return in the "except" block.
# So we treat the "return" separate from the other statements # So we treat the "return" separate from the other statements
cond_except_stmt ::= except_cond1 except_stmts cond_except_stmt ::= except_cond1 except_stmts
cond_except_stmts_opt ::= cond_except_stmt* cond_except_stmts_opt ::= cond_except_stmt*
try_except38r2 ::= SETUP_FINALLY try_except38r2 ::= SETUP_FINALLY
suite_stmts_opt suite_stmts_opt
POP_BLOCK JUMP_FORWARD POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
cond_except_stmts_opt cond_except_stmts_opt
POP_EXCEPT return POP_EXCEPT return
END_FINALLY END_FINALLY
COME_FROM COME_FROM
try_except38r3 ::= SETUP_FINALLY try_except38r3 ::= SETUP_FINALLY
suite_stmts_opt suite_stmts_opt
POP_BLOCK JUMP_FORWARD POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY COME_FROM_FINALLY
cond_except_stmts_opt cond_except_stmts_opt
POP_EXCEPT return POP_EXCEPT return
COME_FROM COME_FROM
END_FINALLY END_FINALLY
COME_FROM COME_FROM
try_except38r4 ::= SETUP_FINALLY try_except38r4 ::= SETUP_FINALLY
returns_in_except returns_in_except
COME_FROM_FINALLY COME_FROM_FINALLY
except_cond1 except_cond1
return return
COME_FROM COME_FROM
END_FINALLY END_FINALLY
# suite_stmts has a return # suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b except_handler38b
try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler_as END_FINALLY COME_FROM except_handler_as END_FINALLY COME_FROM
try_except_as ::= SETUP_FINALLY suite_stmts try_except_as ::= SETUP_FINALLY suite_stmts
except_handler_as END_FINALLY COME_FROM except_handler_as END_FINALLY COME_FROM
try_except_ret38 ::= SETUP_FINALLY returns except_ret38a try_except_ret38 ::= SETUP_FINALLY returns except_ret38a
try_except_ret38a ::= SETUP_FINALLY returns except_handler38c try_except_ret38a ::= SETUP_FINALLY returns except_handler38c
END_FINALLY come_from_opt END_FINALLY come_from_opt
# Note: there is a suite_stmts_opt which seems # Note: there is a suite_stmts_opt which seems
# to be bookkeeping which is not expressed in source code # to be bookkeeping which is not expressed in source code
except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
CALL_FINALLY RETURN_VALUE COME_FROM CALL_FINALLY RETURN_VALUE COME_FROM
COME_FROM_FINALLY COME_FROM_FINALLY
suite_stmts_opt END_FINALLY suite_stmts_opt END_FINALLY
except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
expr ROT_FOUR expr ROT_FOUR
POP_EXCEPT RETURN_VALUE END_FINALLY POP_EXCEPT RETURN_VALUE END_FINALLY
except_handler38 ::= _jump COME_FROM_FINALLY except_handler38 ::= _jump COME_FROM_FINALLY
except_stmts END_FINALLY opt_come_from_except except_stmts END_FINALLY opt_come_from_except
except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
POP_EXCEPT POP_TOP stmts END_FINALLY POP_EXCEPT POP_TOP stmts END_FINALLY
except_handler38c ::= COME_FROM_FINALLY except_cond1a except_stmts except_handler38c ::= COME_FROM_FINALLY except_cond1a except_stmts
POP_EXCEPT JUMP_FORWARD COME_FROM POP_EXCEPT JUMP_FORWARD COME_FROM
except_handler_as ::= COME_FROM_FINALLY except_cond_as tryfinallystmt except_handler_as ::= COME_FROM_FINALLY except_cond_as tryfinallystmt
POP_EXCEPT JUMP_FORWARD COME_FROM POP_EXCEPT JUMP_FORWARD COME_FROM
tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
END_FINALLY END_FINALLY
lc_setup_finally ::= LOAD_CONST SETUP_FINALLY lc_setup_finally ::= LOAD_CONST SETUP_FINALLY
call_finally_pt ::= CALL_FINALLY POP_TOP call_finally_pt ::= CALL_FINALLY POP_TOP
cf_cf_finally ::= come_from_opt COME_FROM_FINALLY cf_cf_finally ::= come_from_opt COME_FROM_FINALLY
pop_finally_pt ::= POP_FINALLY POP_TOP pop_finally_pt ::= POP_FINALLY POP_TOP
ss_end_finally ::= suite_stmts END_FINALLY ss_end_finally ::= suite_stmts END_FINALLY
sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
# FIXME: DRY rules below # FIXME: DRY rules below
tryfinally38rstmt ::= sf_pb_call_returns tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally cf_cf_finally
ss_end_finally ss_end_finally
tryfinally38rstmt ::= sf_pb_call_returns tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally END_FINALLY cf_cf_finally END_FINALLY
suite_stmts suite_stmts
tryfinally38rstmt ::= sf_pb_call_returns tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally POP_FINALLY cf_cf_finally POP_FINALLY
ss_end_finally ss_end_finally
tryfinally38rstmt ::= sf_bp_call_returns tryfinally38rstmt ::= sf_bp_call_returns
COME_FROM_FINALLY POP_FINALLY COME_FROM_FINALLY POP_FINALLY
ss_end_finally ss_end_finally
tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
returns returns
cf_cf_finally pop_finally_pt cf_cf_finally pop_finally_pt
ss_end_finally POP_TOP ss_end_finally POP_TOP
tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
COME_FROM COME_FROM_FINALLY COME_FROM COME_FROM_FINALLY
ss_end_finally ss_end_finally
tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY BEGIN_FINALLY COME_FROM_FINALLY
POP_FINALLY suite_stmts_opt END_FINALLY POP_FINALLY suite_stmts_opt END_FINALLY
tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY BEGIN_FINALLY COME_FROM_FINALLY
POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
""" """
def p_38walrus(self, args): def p_38walrus(self, args):
@@ -362,13 +365,24 @@ class Python38Parser(Python37Parser):
""" """
) )
def customize_grammar_rules(self, tokens, customize): def customize_reduce_checks_full38(self, tokens, customize):
super(Python37Parser, self).customize_grammar_rules(tokens, customize) """
Extra tests when a reduction is made in the full grammar.
Reductions here are extended from those used in the lambda grammar
"""
self.remove_rules_38() self.remove_rules_38()
self.check_reduce["pop_return"] = "tokens"
self.check_reduce["whileTruestmt38"] = "tokens" self.check_reduce["whileTruestmt38"] = "tokens"
self.check_reduce["whilestmt38"] = "tokens" self.check_reduce["whilestmt38"] = "tokens"
self.check_reduce["try_elsestmtl38"] = "AST" self.check_reduce["try_elsestmtl38"] = "AST"
self.reduce_check_table["pop_return"] = pop_return_check
def customize_grammar_rules(self, tokens, customize):
super(Python37Parser, self).customize_grammar_rules(tokens, customize)
self.customize_reduce_checks_full38(tokens, customize)
# For a rough break out on the first word. This may # For a rough break out on the first word. This may
# include instructions that don't need customization, # include instructions that don't need customization,
# but we'll do a finer check after the rough breakout. # but we'll do a finer check after the rough breakout.
@@ -423,11 +437,7 @@ class Python38Parser(Python37Parser):
# Determine if we have an iteration CALL_FUNCTION_1. # Determine if we have an iteration CALL_FUNCTION_1.
has_get_iter_call_function1 = False has_get_iter_call_function1 = False
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
if ( if token == "GET_ITER" and i < n - 2 and tokens[i + 1] == "CALL_FUNCTION_1":
token == "GET_ITER"
and i < n - 2
and tokens[i + 1] == "CALL_FUNCTION_1"
):
has_get_iter_call_function1 = True has_get_iter_call_function1 = True
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
@@ -525,26 +535,26 @@ class Python38Parser(Python37Parser):
continue continue
elif opname == "BUILD_STRING_2": elif opname == "BUILD_STRING_2":
self.addRule( self.addRule(
""" """
expr ::= formatted_value_debug expr ::= formatted_value_debug
formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2 formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2 formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
""", """,
nop_func, nop_func,
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "BUILD_STRING_3": elif opname == "BUILD_STRING_3":
self.addRule( self.addRule(
""" """
expr ::= formatted_value_debug expr ::= formatted_value_debug
formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3 formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3
formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3 formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3
""", """,
nop_func, nop_func,
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "LOAD_CLOSURE": elif opname == "LOAD_CLOSURE":
self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func) self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
@@ -577,14 +587,10 @@ class Python38Parser(Python37Parser):
""" """
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
def reduce_is_invalid(self, rule, ast, tokens, first, last): def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python38Parser, invalid = super(Python38Parser, self).reduce_is_invalid(
self).reduce_is_invalid(rule, ast, rule, ast, tokens, first, last
tokens, first, last) )
self.remove_rules_38() self.remove_rules_38()
if invalid: if invalid:
return invalid return invalid
@@ -612,7 +618,7 @@ if __name__ == "__main__":
p = Python38Parser() p = Python38Parser()
p.remove_rules_38() p.remove_rules_38()
p.check_grammar() p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 8): if PYTHON_VERSION_TRIPLE[:2] == (3, 8):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
@@ -635,7 +641,9 @@ if __name__ == "__main__":
remain_tokens = set(remain_tokens) - opcode_set remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens) print(remain_tokens)
import sys import sys
if len(sys.argv) > 1: if len(sys.argv) > 1:
from spark_parser.spark import rule2str from spark_parser.spark import rule2str
for rule in sorted(p.rule2name.items()): for rule in sorted(p.rule2name.items()):
print(rule2str(rule[0])) print(rule2str(rule[0]))

10
uncompyle6/parsers/reducecheck/pop_return.py Normal file
View File

@@ -0,0 +1,10 @@
# Copyright (c) 2020 Rocky Bernstein
def pop_return_check(
self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
) -> bool:
# If the first instruction of return_expr (the instruction after POP_TOP) is
# has a linestart, then the POP_TOP was probably part of the previous
# statement, such as a call() where the return value is discarded.
return tokens[first + 1].linestart

13
uncompyle6/scanner.py
View File

@@ -22,6 +22,7 @@ scanners, e.g. for Python 2.7 or 3.4.
""" """
import sys import sys
from abc import ABC
from array import array from array import array
from collections import namedtuple from collections import namedtuple
@@ -89,7 +90,7 @@ def long(num):
CONST_COLLECTIONS = ("CONST_LIST", "CONST_SET", "CONST_DICT", "CONST_MAP") CONST_COLLECTIONS = ("CONST_LIST", "CONST_SET", "CONST_DICT", "CONST_MAP")
class Code(object): class Code:
""" """
Class for representing code-objects. Class for representing code-objects.
@@ -108,7 +109,7 @@ class Code(object):
self._tokens, self._customize = scanner.ingest(co, classname, show_asm=show_asm) self._tokens, self._customize = scanner.ingest(co, classname, show_asm=show_asm)
class Scanner: class Scanner(ABC):
def __init__(self, version: tuple, show_asm=None, is_pypy=False): def __init__(self, version: tuple, show_asm=None, is_pypy=False):
self.version = version self.version = version
self.show_asm = show_asm self.show_asm = show_asm
@@ -292,7 +293,13 @@ class Scanner:
return False return False
return offset < self.get_target(offset) return offset < self.get_target(offset)
def prev_offset(self, offset): def ingest(self, co, classname=None, code_objects={}, show_asm=None):
"""
Code to tokenize disassembly. Subclasses must implement this.
"""
raise NotImplementedError("This method should have been implemented")
def prev_offset(self, offset: int) -> int:
return self.insts[self.offset2inst_index[offset] - 1].offset return self.insts[self.offset2inst_index[offset] - 1].offset
def get_inst(self, offset): def get_inst(self, offset):

2
uncompyle6/scanners/scanner37.py
View File

@@ -52,7 +52,7 @@ class Scanner37(Scanner37Base):
if collection_type == "CONST_DICT": if collection_type == "CONST_DICT":
# constant dictionaries work via BUILD_CONST_KEY_MAP and # constant dictionaries work via BUILD_CONST_KEY_MAP and
# handle the values() like sets and lists. # handle the values() like sets and lists.
# However the keys() are an LOAD_CONST of the keys. # However, the keys() are an LOAD_CONST of the keys.
# adjust offset to account for this # adjust offset to account for this
count += 1 count += 1

7
uncompyle6/scanners/scanner37base.py
View File

@@ -263,10 +263,9 @@ class Scanner37Base(Scanner):
if ( if (
next_inst.opname == "LOAD_GLOBAL" next_inst.opname == "LOAD_GLOBAL"
and next_inst.argval == "AssertionError" and next_inst.argval == "AssertionError"
and inst.argval and inst.argval is not None
): ):
raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] raise_inst = self.get_inst(self.prev_op[inst.argval])
raise_inst = self.insts[raise_idx]
if raise_inst.opname.startswith("RAISE_VARARGS"): if raise_inst.opname.startswith("RAISE_VARARGS"):
self.load_asserts.add(next_inst.offset) self.load_asserts.add(next_inst.offset)
pass pass
@@ -283,7 +282,7 @@ class Scanner37Base(Scanner):
# some backward jumps, are turned into forward jumps to another # some backward jumps, are turned into forward jumps to another
# "extended arg" backward jump to the same location. # "extended arg" backward jump to the same location.
if inst.opname == "JUMP_FORWARD": if inst.opname == "JUMP_FORWARD":
jump_inst = self.insts[self.offset2inst_index[inst.argval]] jump_inst = self.get_inst(inst.argval)
if jump_inst.has_extended_arg and jump_inst.opname.startswith("JUMP"): if jump_inst.has_extended_arg and jump_inst.opname.startswith("JUMP"):
# Create a combination of the jump-to instruction and # Create a combination of the jump-to instruction and
# this one. Keep the position information of this instruction, # this one. Keep the position information of this instruction,

38
uncompyle6/semantics/gencomp.py
View File

@@ -21,9 +21,9 @@ from xdis import co_flags_is_async, iscode
from uncompyle6.parser import get_python_parser from uncompyle6.parser import get_python_parser
from uncompyle6.scanner import Code from uncompyle6.scanner import Code
from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.consts import PRECEDENCE from uncompyle6.semantics.consts import PRECEDENCE
from uncompyle6.semantics.helper import is_lambda_mode from uncompyle6.semantics.helper import is_lambda_mode
from uncompyle6.scanners.tok import Token
class ComprehensionMixin: class ComprehensionMixin:
@@ -172,7 +172,10 @@ class ComprehensionMixin:
tree = tree[1] tree = tree[1]
pass pass
if tree in ("genexpr_func", "genexpr_func_async",): if tree in (
"genexpr_func",
"genexpr_func_async",
):
for i in range(3, 5): for i in range(3, 5):
if tree[i] == "comp_iter": if tree[i] == "comp_iter":
iter_index = i iter_index = i
@@ -330,8 +333,19 @@ class ComprehensionMixin:
assert store == "store" assert store == "store"
n = set_iter_async[2] n = set_iter_async[2]
elif node == "list_comp" and tree[0] == "expr": elif node == "list_comp" and tree[0] == "expr":
tree = tree[0][0] list_iter = None
n = tree[iter_index] for list_iter_try in tree:
if list_iter_try == "list_iter":
list_iter = list_iter_try
break
if not list_iter_try:
tree = tree[0][0]
n = tree[iter_index]
else:
n = list_iter
pass
pass
pass
else: else:
n = tree[iter_index] n = tree[iter_index]
@@ -405,6 +419,9 @@ class ComprehensionMixin:
n = n[0] n = n[0]
if n in ("list_for", "comp_for"): if n in ("list_for", "comp_for"):
if n == "list_for" and not comp_for and n[0] == "expr":
comp_for = n[0]
n_index = 3 n_index = 3
if ( if (
(n[2] == "store") (n[2] == "store")
@@ -494,11 +511,21 @@ class ComprehensionMixin:
if comp_for: if comp_for:
self.preorder(comp_for) self.preorder(comp_for)
else: else:
try:
node[in_node_index]
except:
from trepan.api import debug
debug()
self.preorder(node[in_node_index]) self.preorder(node[in_node_index])
# Here is where we handle nested list iterations. # Here is where we handle nested list iterations.
if tree == "list_comp" and self.version != (3, 0): if tree == "list_comp" and self.version != (3, 0):
list_iter = tree[1] list_iter = None
for list_iter_try in tree:
if list_iter_try == "list_iter":
list_iter = list_iter_try
break
assert list_iter == "list_iter" assert list_iter == "list_iter"
if list_iter[0] == "list_for": if list_iter[0] == "list_for":
self.preorder(list_iter[0][3]) self.preorder(list_iter[0][3])
@@ -637,7 +664,6 @@ class ComprehensionMixin:
# Find the list comprehension body. It is the inner-most # Find the list comprehension body. It is the inner-most
# node that is not list_.. . # node that is not list_.. .
while n == "list_iter": while n == "list_iter":
# recurse one step # recurse one step
n = n[0] n = n[0]

2
uncompyle6/semantics/n_actions.py
View File

@@ -1036,7 +1036,7 @@ class NonterminalActions:
self.prec = p self.prec = p
self.prune() # stop recursing self.prune() # stop recursing
def n_listcomp(self, node): def n_list_comp(self, node):
self.write("[") self.write("[")
if node[0].kind == "load_closure": if node[0].kind == "load_closure":
assert self.version >= (3, 0) assert self.version >= (3, 0)