diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index dc304126..9e206c3c 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -6,7 +6,7 @@ open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username +liberapay: rocky issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/test/stdlib/3.2-exclude.sh b/test/stdlib/3.2-exclude.sh index f80f9ee8..87aec9aa 100644 --- a/test/stdlib/3.2-exclude.sh +++ b/test/stdlib/3.2-exclude.sh @@ -1,7 +1,14 @@ SKIP_TESTS=( - [test_descr.py]=1 # FIXME: Works on c90ff51? + [test_descr.py]=1 + # [test_descr.py]=pytest_module # FIXME: Works on c90ff51? + # AssertionError: 'D(4)C(4)A(4)' != 'D(4)C(4)B(4)A(4)' + # - D(4)C(4)A(4) + # + D(4)C(4)B(4)A(4) + # ? ++++ - [test_cmath.py]=1 # FIXME + + [test_cmath.py]=1 # Control-flow "elif else -> else: if else" + # [test_cmath.py]=pytest_module # AssertionError: rect1000: rect(complex(0.0, 0.0)) # Expected: complex(0.0, 0.0) # Received: complex(0.0, -1.0) @@ -9,14 +16,27 @@ SKIP_TESTS=( [test_cmd_line.py]=1 - [test_collections.py]=1 + + [test_collections.py]=1 # fail on its own + # E TypeError: __new__() takes exactly 4 arguments (1 given) + [test_concurrent_futures.py]=1 # too long to run over 46 seconds by itself - [test_datetimetester.py]=1 - [test_decimal.py]=1 - [test_dictcomps.py]=1 # FIXME: semantic error: actual = {k:v for k in } - [test_doctest.py]=1 # test failures + [test_datetime.py]=pytest_module + + [test_decimal.py]=1 # Fails on its own, even with pytest + + [test_dictcomps.py]=1 + # [test_dictcomps.py]=pytest_module # FIXME: semantic error: actual = {k:v for k in } + # assert (count * 2) <= i + + [test_doctest.py]=1 # Missing pytest fixture + # [test_doctest.py]=pytest_module + # fixture 'coverdir' not found + [test_dis.py]=1 # We change line numbers - duh! + [test_exceptions.py]=1 # parse error + # [test_exceptions.py]=pytest_module # parse error [test_modulefinder.py]=1 # test failures [test_multiprocessing.py]=1 # test takes too long to run: 35 seconds diff --git a/test/stdlib/3.3-exclude.sh b/test/stdlib/3.3-exclude.sh index b2d0afe8..a45c81ca 100644 --- a/test/stdlib/3.3-exclude.sh +++ b/test/stdlib/3.3-exclude.sh @@ -10,7 +10,7 @@ SKIP_TESTS=( # tgt.append(elem) [test_itertools.py]=1 - [test_buffer.py]=1 # FIXME: Works on c90ff51 + [test_buffer.py]=pytest # FIXME: Works on c90ff51 [test_cmath.py]=pytest [test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines diff --git a/test/stdlib/3.8-exclude.sh b/test/stdlib/3.8-exclude.sh index d4393ae9..61f8700c 100644 --- a/test/stdlib/3.8-exclude.sh +++ b/test/stdlib/3.8-exclude.sh @@ -28,12 +28,12 @@ SKIP_TESTS=( # These and the above may be due to new code generation or tests # between 3.8.3 and 3.8.5 ? - [test_decorators.py]=1 # + [test_decorators.py]=1 # parse error - [test_dtrace.py]=1 # - [test_exceptions.py]=1 # + [test_dtrace.py]=1 # parse error + [test_exceptions.py]=1 # parse error [test_ftplib.py]=1 # - [test_gc.py]=1 # + [test_gc.py]=1 # FIXME: return return strip_python_stderr(stderr) [test_gzip.py]=1 # [test_hashlib.py]=1 # [test_iter.py]=1 # @@ -51,7 +51,6 @@ SKIP_TESTS=( [test_audioop.py]=1 # test failure [test_audit.py]=1 # parse error - [test_base64.py]=1 # parse error [test_baseexception.py]=1 # [test_bigaddrspace.py]=1 # parse error [test_bigmem.py]=1 # parse error diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 37dd050c..22719c44 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -168,12 +168,8 @@ if ((IS_PYPY)); then else cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR fi -if [[ $PYVERSION == 3.2 ]] ; then - cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR - cd $TESTDIR -else - cd $TESTDIR/test -fi +cd $TESTDIR/test + pyenv local $FULLVERSION export PYTHONPATH=$TESTDIR export PATH=${PYENV_ROOT}/shims:${PATH} @@ -188,7 +184,7 @@ if [[ -n $1 ]] ; then typeset -a files_ary=( $(echo $@) ) if (( ${#files_ary[@]} == 1 || DONT_SKIP_TESTS == 1 )) ; then for file in $files; do - if (( SKIP_TESTS[$file] != "pytest" )); then + if (( SKIP_TESTS[$file] != "pytest" || SKIP_TESTS[$file] != "pytest_module" )); then SKIP_TESTS[$file]=1; fi done @@ -208,6 +204,8 @@ for file in $files; do if [[ ${SKIP_TESTS[$file]} == "pytest" ]]; then PYTHON=pytest + elif [[ ${SKIP_TESTS[$file]} == "pytest_module" ]]; then + PYTHON="$PYTHON -m pytest" else if [[ ${SKIP_TESTS[$file]}s == ${NOT_INVERTED_TESTS} ]] ; then ((skipped++)) diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 86cf8bd4..6b871968 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2021 Rocky Bernstein +# Copyright (c) 2015-2021, 2024 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # Copyright (c) 1999 John Aycock @@ -27,11 +27,12 @@ that a later phase can turn into a sequence of ASCII text. from __future__ import print_function -from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt -from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func -from uncompyle6.parsers.treenode import SyntaxTree from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func +from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt +from uncompyle6.parsers.treenode import SyntaxTree + class Python2Parser(PythonParser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): @@ -405,7 +406,6 @@ class Python2Parser(PythonParser): "CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_KW", ): - args_pos, args_kw = self.get_pos_kw(token) # number of apply equiv arguments: @@ -526,7 +526,7 @@ class Python2Parser(PythonParser): custom_seen_ops.add(opname) continue elif opname == "LOAD_LISTCOMP": - self.addRule("expr ::= listcomp", nop_func) + self.addRule("expr ::= list_comp", nop_func) custom_seen_ops.add(opname) continue elif opname == "LOAD_SETCOMP": diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 64c164d4..70c984ac 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1085,7 +1085,9 @@ class Python3Parser(PythonParser): ) custom_ops_processed.add(opname) elif opname == "LOAD_LISTCOMP": - self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) + self.add_unique_rule( + "expr ::= list_comp", opname, token.attr, customize + ) custom_ops_processed.add(opname) elif opname == "LOAD_SETCOMP": # Should this be generalized and put under MAKE_FUNCTION? @@ -1154,7 +1156,7 @@ class Python3Parser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) ) @@ -1348,14 +1350,14 @@ class Python3Parser(PythonParser): # 'exprs' in the rule above into a # tuple. rule_pat = ( - "listcomp ::= load_closure LOAD_LISTCOMP %%s%s " + "list_comp ::= load_closure LOAD_LISTCOMP %%s%s " "expr GET_ITER CALL_FUNCTION_1" % (opname,) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize ) rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) ) @@ -1399,7 +1401,7 @@ class Python3Parser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) ) diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index a3942f5f..cf0c154b 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -723,7 +723,9 @@ class Python37BaseParser(PythonParser): ) custom_ops_processed.add(opname) elif opname == "LOAD_LISTCOMP": - self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) + self.add_unique_rule( + "expr ::= list_comp", opname, token.attr, customize + ) custom_ops_processed.add(opname) elif opname == "LOAD_NAME": if ( @@ -802,7 +804,7 @@ class Python37BaseParser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * args_pos, opname) ) @@ -900,14 +902,14 @@ class Python37BaseParser(PythonParser): # 'exprs' in the rule above into a # tuple. rule_pat = ( - "listcomp ::= load_closure LOAD_LISTCOMP %%s%s " + "list_comp ::= load_closure LOAD_LISTCOMP %%s%s " "expr GET_ITER CALL_FUNCTION_1" % (opname,) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize ) rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) ) self.add_make_function_rule( @@ -941,7 +943,7 @@ class Python37BaseParser(PythonParser): # and have GET_ITER CALL_FUNCTION_1 # Todo: For Pypy we need to modify this slightly rule_pat = ( - "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + "list_comp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname) ) self.add_make_function_rule( diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index 1a638c01..10367419 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2020, 2022-2023 Rocky Bernstein +# Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,269 +17,272 @@ spark grammar differences over Python 3.7 for Python 3.8 """ from __future__ import print_function -from uncompyle6.parser import PythonParserSingle, nop_func from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG + +from uncompyle6.parser import PythonParserSingle, nop_func from uncompyle6.parsers.parse37 import Python37Parser +from uncompyle6.parsers.reducecheck.pop_return import pop_return_check + class Python38Parser(Python37Parser): def p_38_stmt(self, args): """ - stmt ::= async_for_stmt38 - stmt ::= async_forelse_stmt38 - stmt ::= call_stmt - stmt ::= continue - stmt ::= for38 - stmt ::= forelselaststmt38 - stmt ::= forelselaststmtl38 - stmt ::= forelsestmt38 - stmt ::= try_elsestmtl38 - stmt ::= try_except38 - stmt ::= try_except38r - stmt ::= try_except38r2 - stmt ::= try_except38r3 - stmt ::= try_except38r4 - stmt ::= try_except_as - stmt ::= try_except_ret38 - stmt ::= tryfinally38astmt - stmt ::= tryfinally38rstmt - stmt ::= tryfinally38rstmt2 - stmt ::= tryfinally38rstmt3 - stmt ::= tryfinally38stmt - stmt ::= whileTruestmt38 - stmt ::= whilestmt38 + stmt ::= async_for_stmt38 + stmt ::= async_forelse_stmt38 + stmt ::= call_stmt + stmt ::= continue + stmt ::= for38 + stmt ::= forelselaststmt38 + stmt ::= forelselaststmtl38 + stmt ::= forelsestmt38 + stmt ::= try_elsestmtl38 + stmt ::= try_except38 + stmt ::= try_except38r + stmt ::= try_except38r2 + stmt ::= try_except38r3 + stmt ::= try_except38r4 + stmt ::= try_except_as + stmt ::= try_except_ret38 + stmt ::= tryfinally38astmt + stmt ::= tryfinally38rstmt + stmt ::= tryfinally38rstmt2 + stmt ::= tryfinally38rstmt3 + stmt ::= tryfinally38stmt + stmt ::= whileTruestmt38 + stmt ::= whilestmt38 - call_stmt ::= call - break ::= POP_BLOCK BREAK_LOOP - break ::= POP_BLOCK POP_TOP BREAK_LOOP - break ::= POP_TOP BREAK_LOOP - break ::= POP_EXCEPT BREAK_LOOP + call_stmt ::= call + break ::= POP_BLOCK BREAK_LOOP + break ::= POP_BLOCK POP_TOP BREAK_LOOP + break ::= POP_TOP BREAK_LOOP + break ::= POP_EXCEPT BREAK_LOOP - # The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed. - # Inside an loop we can have this, which can only appear in side a try/except - # And it can also appear at the end of the try except. - continue ::= POP_EXCEPT JUMP_BACK + # The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed. + # Inside an loop we can have this, which can only appear in side a try/except + # And it can also appear at the end of the try except. + continue ::= POP_EXCEPT JUMP_BACK - # FIXME: this should be restricted to being inside a try block - stmt ::= except_ret38 - stmt ::= except_ret38a + # FIXME: this should be restricted to being inside a try block + stmt ::= except_ret38 + stmt ::= except_ret38a - # FIXME: this should be added only when seeing GET_AITER or YIELD_FROM - async_for ::= GET_AITER _come_froms - SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK - async_for_stmt38 ::= expr async_for - store for_block - COME_FROM_FINALLY - END_ASYNC_FOR + # FIXME: this should be added only when seeing GET_AITER or YIELD_FROM + async_for ::= GET_AITER _come_froms + SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK + async_for_stmt38 ::= expr async_for + store for_block + COME_FROM_FINALLY + END_ASYNC_FOR - genexpr_func_async ::= LOAD_ARG func_async_prefix - store comp_iter - JUMP_BACK COME_FROM_FINALLY - END_ASYNC_FOR + genexpr_func_async ::= LOAD_ARG func_async_prefix + store comp_iter + JUMP_BACK COME_FROM_FINALLY + END_ASYNC_FOR - # FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of - # for / forelse is used. Add come froms and check of add up control-flow detection phase. - async_forelse_stmt38 ::= expr - GET_AITER - SETUP_FINALLY - GET_ANEXT - LOAD_CONST - YIELD_FROM - POP_BLOCK - store for_block - COME_FROM_FINALLY - END_ASYNC_FOR - else_suite - - # Seems to be used to discard values before a return in a "for" loop - discard_top ::= ROT_TWO POP_TOP - discard_tops ::= discard_top+ - - return ::= return_expr - discard_tops RETURN_VALUE - - return ::= popb_return - return ::= pop_return - return ::= pop_ex_return - except_stmt ::= pop_ex_return - pop_return ::= POP_TOP return_expr RETURN_VALUE - popb_return ::= return_expr POP_BLOCK RETURN_VALUE - pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE - - # 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it - lastl_stmt ::= ifpoplaststmtl - ifpoplaststmtl ::= testexpr POP_TOP c_stmts_opt - ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms - - # Keep indices the same in ifelsestmtl - cf_pt ::= COME_FROM POP_TOP - ifelsestmtl ::= testexpr c_stmts cf_pt else_suite - - for38 ::= expr get_iter store for_block JUMP_BACK - for38 ::= expr get_for_iter store for_block JUMP_BACK - for38 ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK - for38 ::= expr get_for_iter store for_block - - forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite - forelsestmt38 ::= expr get_for_iter store for_block JUMP_BACK _come_froms - else_suite - - forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec - forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel - - returns_in_except ::= _stmts except_return_value - except_return_value ::= POP_BLOCK return - except_return_value ::= expr POP_BLOCK RETURN_VALUE - - whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK + # FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of + # for / forelse is used. Add come froms and check of add up control-flow detection phase. + async_forelse_stmt38 ::= expr + GET_AITER + SETUP_FINALLY + GET_ANEXT + LOAD_CONST + YIELD_FROM POP_BLOCK - whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK - whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms - whilestmt38 ::= _come_froms testexpr returns POP_BLOCK - whilestmt38 ::= _come_froms testexpr l_stmts JUMP_BACK - whilestmt38 ::= _come_froms testexpr l_stmts come_froms + store for_block + COME_FROM_FINALLY + END_ASYNC_FOR + else_suite - # while1elsestmt ::= l_stmts JUMP_BACK - whileTruestmt ::= _come_froms l_stmts JUMP_BACK POP_BLOCK - while1stmt ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP - whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK - whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE - whileTruestmt38 ::= _come_froms pass JUMP_BACK + # Seems to be used to discard values before a return in a "for" loop + discard_top ::= ROT_TWO POP_TOP + discard_tops ::= discard_top+ - for_block ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK + return ::= return_expr + discard_tops RETURN_VALUE - except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false - POP_TOP POP_TOP POP_TOP - POP_EXCEPT - except_cond_as ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE - POP_TOP STORE_FAST POP_TOP + return ::= popb_return + return ::= pop_return + return ::= pop_ex_return + except_stmt ::= pop_ex_return + pop_return ::= POP_TOP return_expr RETURN_VALUE + popb_return ::= return_expr POP_BLOCK RETURN_VALUE + pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE - try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK - except_handler38 COME_FROM - else_suitel opt_come_from_except - try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK - except_handler38 + # 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it + lastl_stmt ::= ifpoplaststmtl + ifpoplaststmtl ::= testexpr POP_TOP c_stmts_opt + ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms - try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt - except_handler38a + # Keep indices the same in ifelsestmtl + cf_pt ::= COME_FROM POP_TOP + ifelsestmtl ::= testexpr c_stmts cf_pt else_suite - # suite_stmts has a return - try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts - except_handler38b - try_except38r ::= SETUP_FINALLY return_except - except_handler38b - return_except ::= stmts POP_BLOCK return + for38 ::= expr get_iter store for_block JUMP_BACK + for38 ::= expr get_for_iter store for_block JUMP_BACK + for38 ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK + for38 ::= expr get_for_iter store for_block + + forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite + forelsestmt38 ::= expr get_for_iter store for_block JUMP_BACK _come_froms + else_suite + + forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel + + returns_in_except ::= _stmts except_return_value + except_return_value ::= POP_BLOCK return + except_return_value ::= expr POP_BLOCK RETURN_VALUE + + whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK + POP_BLOCK + whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms + whilestmt38 ::= _come_froms testexpr returns POP_BLOCK + whilestmt38 ::= _come_froms testexpr l_stmts JUMP_BACK + whilestmt38 ::= _come_froms testexpr l_stmts come_froms + + # while1elsestmt ::= l_stmts JUMP_BACK + whileTruestmt ::= _come_froms l_stmts JUMP_BACK POP_BLOCK + while1stmt ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK + whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE + whileTruestmt38 ::= _come_froms pass JUMP_BACK + + for_block ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK + + except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false + POP_TOP POP_TOP POP_TOP + POP_EXCEPT + except_cond_as ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE + POP_TOP STORE_FAST POP_TOP + + try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + except_handler38 COME_FROM + else_suitel opt_come_from_except + try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + except_handler38 + + try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt + except_handler38a + + # suite_stmts has a return + try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts + except_handler38b + try_except38r ::= SETUP_FINALLY return_except + except_handler38b + return_except ::= stmts POP_BLOCK return - # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there - # is a final return in the "except" block. - # So we treat the "return" separate from the other statements - cond_except_stmt ::= except_cond1 except_stmts - cond_except_stmts_opt ::= cond_except_stmt* + # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there + # is a final return in the "except" block. + # So we treat the "return" separate from the other statements + cond_except_stmt ::= except_cond1 except_stmts + cond_except_stmts_opt ::= cond_except_stmt* - try_except38r2 ::= SETUP_FINALLY - suite_stmts_opt - POP_BLOCK JUMP_FORWARD - COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP - cond_except_stmts_opt - POP_EXCEPT return - END_FINALLY - COME_FROM + try_except38r2 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + cond_except_stmts_opt + POP_EXCEPT return + END_FINALLY + COME_FROM - try_except38r3 ::= SETUP_FINALLY - suite_stmts_opt - POP_BLOCK JUMP_FORWARD - COME_FROM_FINALLY - cond_except_stmts_opt - POP_EXCEPT return - COME_FROM - END_FINALLY - COME_FROM + try_except38r3 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY + cond_except_stmts_opt + POP_EXCEPT return + COME_FROM + END_FINALLY + COME_FROM - try_except38r4 ::= SETUP_FINALLY - returns_in_except - COME_FROM_FINALLY - except_cond1 - return - COME_FROM - END_FINALLY + try_except38r4 ::= SETUP_FINALLY + returns_in_except + COME_FROM_FINALLY + except_cond1 + return + COME_FROM + END_FINALLY - # suite_stmts has a return - try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts - except_handler38b - try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts - except_handler_as END_FINALLY COME_FROM - try_except_as ::= SETUP_FINALLY suite_stmts - except_handler_as END_FINALLY COME_FROM + # suite_stmts has a return + try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts + except_handler38b + try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts + except_handler_as END_FINALLY COME_FROM + try_except_as ::= SETUP_FINALLY suite_stmts + except_handler_as END_FINALLY COME_FROM - try_except_ret38 ::= SETUP_FINALLY returns except_ret38a - try_except_ret38a ::= SETUP_FINALLY returns except_handler38c - END_FINALLY come_from_opt + try_except_ret38 ::= SETUP_FINALLY returns except_ret38a + try_except_ret38a ::= SETUP_FINALLY returns except_handler38c + END_FINALLY come_from_opt - # Note: there is a suite_stmts_opt which seems - # to be bookkeeping which is not expressed in source code - except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT - CALL_FINALLY RETURN_VALUE COME_FROM - COME_FROM_FINALLY - suite_stmts_opt END_FINALLY - except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP - expr ROT_FOUR - POP_EXCEPT RETURN_VALUE END_FINALLY + # Note: there is a suite_stmts_opt which seems + # to be bookkeeping which is not expressed in source code + except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT + CALL_FINALLY RETURN_VALUE COME_FROM + COME_FROM_FINALLY + suite_stmts_opt END_FINALLY + except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + expr ROT_FOUR + POP_EXCEPT RETURN_VALUE END_FINALLY - except_handler38 ::= _jump COME_FROM_FINALLY - except_stmts END_FINALLY opt_come_from_except - except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP - POP_EXCEPT POP_TOP stmts END_FINALLY + except_handler38 ::= _jump COME_FROM_FINALLY + except_stmts END_FINALLY opt_come_from_except + except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + POP_EXCEPT POP_TOP stmts END_FINALLY - except_handler38c ::= COME_FROM_FINALLY except_cond1a except_stmts - POP_EXCEPT JUMP_FORWARD COME_FROM - except_handler_as ::= COME_FROM_FINALLY except_cond_as tryfinallystmt - POP_EXCEPT JUMP_FORWARD COME_FROM + except_handler38c ::= COME_FROM_FINALLY except_cond1a except_stmts + POP_EXCEPT JUMP_FORWARD COME_FROM + except_handler_as ::= COME_FROM_FINALLY except_cond_as tryfinallystmt + POP_EXCEPT JUMP_FORWARD COME_FROM - tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK - BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt - END_FINALLY + tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt + END_FINALLY - lc_setup_finally ::= LOAD_CONST SETUP_FINALLY - call_finally_pt ::= CALL_FINALLY POP_TOP - cf_cf_finally ::= come_from_opt COME_FROM_FINALLY - pop_finally_pt ::= POP_FINALLY POP_TOP - ss_end_finally ::= suite_stmts END_FINALLY - sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns + lc_setup_finally ::= LOAD_CONST SETUP_FINALLY + call_finally_pt ::= CALL_FINALLY POP_TOP + cf_cf_finally ::= come_from_opt COME_FROM_FINALLY + pop_finally_pt ::= POP_FINALLY POP_TOP + ss_end_finally ::= suite_stmts END_FINALLY + sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns - # FIXME: DRY rules below - tryfinally38rstmt ::= sf_pb_call_returns - cf_cf_finally - ss_end_finally - tryfinally38rstmt ::= sf_pb_call_returns - cf_cf_finally END_FINALLY - suite_stmts - tryfinally38rstmt ::= sf_pb_call_returns - cf_cf_finally POP_FINALLY - ss_end_finally - tryfinally38rstmt ::= sf_bp_call_returns - COME_FROM_FINALLY POP_FINALLY - ss_end_finally + # FIXME: DRY rules below + tryfinally38rstmt ::= sf_pb_call_returns + cf_cf_finally + ss_end_finally + tryfinally38rstmt ::= sf_pb_call_returns + cf_cf_finally END_FINALLY + suite_stmts + tryfinally38rstmt ::= sf_pb_call_returns + cf_cf_finally POP_FINALLY + ss_end_finally + tryfinally38rstmt ::= sf_bp_call_returns + COME_FROM_FINALLY POP_FINALLY + ss_end_finally - tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt - returns - cf_cf_finally pop_finally_pt - ss_end_finally POP_TOP - tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE - COME_FROM COME_FROM_FINALLY - ss_end_finally + tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt + returns + cf_cf_finally pop_finally_pt + ss_end_finally POP_TOP + tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE + COME_FROM COME_FROM_FINALLY + ss_end_finally - tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK - BEGIN_FINALLY COME_FROM_FINALLY - POP_FINALLY suite_stmts_opt END_FINALLY + tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY + POP_FINALLY suite_stmts_opt END_FINALLY - tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK - BEGIN_FINALLY COME_FROM_FINALLY - POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP + tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_FINALLY + POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP """ def p_38walrus(self, args): @@ -362,13 +365,24 @@ class Python38Parser(Python37Parser): """ ) - def customize_grammar_rules(self, tokens, customize): - super(Python37Parser, self).customize_grammar_rules(tokens, customize) + def customize_reduce_checks_full38(self, tokens, customize): + """ + Extra tests when a reduction is made in the full grammar. + + Reductions here are extended from those used in the lambda grammar + """ self.remove_rules_38() + self.check_reduce["pop_return"] = "tokens" self.check_reduce["whileTruestmt38"] = "tokens" self.check_reduce["whilestmt38"] = "tokens" self.check_reduce["try_elsestmtl38"] = "AST" + self.reduce_check_table["pop_return"] = pop_return_check + + def customize_grammar_rules(self, tokens, customize): + super(Python37Parser, self).customize_grammar_rules(tokens, customize) + self.customize_reduce_checks_full38(tokens, customize) + # For a rough break out on the first word. This may # include instructions that don't need customization, # but we'll do a finer check after the rough breakout. @@ -423,11 +437,7 @@ class Python38Parser(Python37Parser): # Determine if we have an iteration CALL_FUNCTION_1. has_get_iter_call_function1 = False for i, token in enumerate(tokens): - if ( - token == "GET_ITER" - and i < n - 2 - and tokens[i + 1] == "CALL_FUNCTION_1" - ): + if token == "GET_ITER" and i < n - 2 and tokens[i + 1] == "CALL_FUNCTION_1": has_get_iter_call_function1 = True for i, token in enumerate(tokens): @@ -525,26 +535,26 @@ class Python38Parser(Python37Parser): continue elif opname == "BUILD_STRING_2": - self.addRule( - """ + self.addRule( + """ expr ::= formatted_value_debug formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2 formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2 """, - nop_func, - ) - custom_ops_processed.add(opname) + nop_func, + ) + custom_ops_processed.add(opname) elif opname == "BUILD_STRING_3": - self.addRule( - """ + self.addRule( + """ expr ::= formatted_value_debug formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3 formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3 """, - nop_func, - ) - custom_ops_processed.add(opname) + nop_func, + ) + custom_ops_processed.add(opname) elif opname == "LOAD_CLOSURE": self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func) @@ -577,14 +587,10 @@ class Python38Parser(Python37Parser): """ self.addRule(rule, nop_func) - - - - def reduce_is_invalid(self, rule, ast, tokens, first, last): - invalid = super(Python38Parser, - self).reduce_is_invalid(rule, ast, - tokens, first, last) + invalid = super(Python38Parser, self).reduce_is_invalid( + rule, ast, tokens, first, last + ) self.remove_rules_38() if invalid: return invalid @@ -612,7 +618,7 @@ if __name__ == "__main__": p = Python38Parser() p.remove_rules_38() p.check_grammar() - from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY + from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE if PYTHON_VERSION_TRIPLE[:2] == (3, 8): lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() @@ -635,7 +641,9 @@ if __name__ == "__main__": remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) import sys + if len(sys.argv) > 1: from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): print(rule2str(rule[0])) diff --git a/uncompyle6/parsers/reducecheck/pop_return.py b/uncompyle6/parsers/reducecheck/pop_return.py new file mode 100644 index 00000000..e9da0264 --- /dev/null +++ b/uncompyle6/parsers/reducecheck/pop_return.py @@ -0,0 +1,10 @@ +# Copyright (c) 2020 Rocky Bernstein + + +def pop_return_check( + self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int +) -> bool: + # If the first instruction of return_expr (the instruction after POP_TOP) is + # has a linestart, then the POP_TOP was probably part of the previous + # statement, such as a call() where the return value is discarded. + return tokens[first + 1].linestart diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 2cbe393f..bda8d972 100644 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -89,7 +89,7 @@ def long(num): CONST_COLLECTIONS = ("CONST_LIST", "CONST_SET", "CONST_DICT", "CONST_MAP") -class Code(object): +class Code: """ Class for representing code-objects. @@ -292,7 +292,13 @@ class Scanner: return False return offset < self.get_target(offset) - def prev_offset(self, offset): + def ingest(self, co, classname=None, code_objects={}, show_asm=None): + """ + Code to tokenize disassembly. Subclasses must implement this. + """ + raise NotImplementedError("This method should have been implemented") + + def prev_offset(self, offset: int) -> int: return self.insts[self.offset2inst_index[offset] - 1].offset def get_inst(self, offset): diff --git a/uncompyle6/scanners/scanner37.py b/uncompyle6/scanners/scanner37.py index 75b74d64..83f8c172 100644 --- a/uncompyle6/scanners/scanner37.py +++ b/uncompyle6/scanners/scanner37.py @@ -52,7 +52,7 @@ class Scanner37(Scanner37Base): if collection_type == "CONST_DICT": # constant dictionaries work via BUILD_CONST_KEY_MAP and # handle the values() like sets and lists. - # However the keys() are an LOAD_CONST of the keys. + # However, the keys() are an LOAD_CONST of the keys. # adjust offset to account for this count += 1 diff --git a/uncompyle6/scanners/scanner37base.py b/uncompyle6/scanners/scanner37base.py index a9ae8151..1def228e 100644 --- a/uncompyle6/scanners/scanner37base.py +++ b/uncompyle6/scanners/scanner37base.py @@ -263,10 +263,9 @@ class Scanner37Base(Scanner): if ( next_inst.opname == "LOAD_GLOBAL" and next_inst.argval == "AssertionError" - and inst.argval + and inst.argval is not None ): - raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] - raise_inst = self.insts[raise_idx] + raise_inst = self.get_inst(self.prev_op[inst.argval]) if raise_inst.opname.startswith("RAISE_VARARGS"): self.load_asserts.add(next_inst.offset) pass @@ -283,7 +282,7 @@ class Scanner37Base(Scanner): # some backward jumps, are turned into forward jumps to another # "extended arg" backward jump to the same location. if inst.opname == "JUMP_FORWARD": - jump_inst = self.insts[self.offset2inst_index[inst.argval]] + jump_inst = self.get_inst(inst.argval) if jump_inst.has_extended_arg and jump_inst.opname.startswith("JUMP"): # Create a combination of the jump-to instruction and # this one. Keep the position information of this instruction, diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index e07e26fe..c9406e24 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -21,9 +21,9 @@ from xdis import co_flags_is_async, iscode from uncompyle6.parser import get_python_parser from uncompyle6.scanner import Code +from uncompyle6.scanners.tok import Token from uncompyle6.semantics.consts import PRECEDENCE from uncompyle6.semantics.helper import is_lambda_mode -from uncompyle6.scanners.tok import Token class ComprehensionMixin: @@ -100,7 +100,7 @@ class ComprehensionMixin: self, node, iter_index, - code_index = -5, + code_index=-5, ): p = self.prec self.prec = PRECEDENCE["lambda_body"] - 1 @@ -172,7 +172,10 @@ class ComprehensionMixin: tree = tree[1] pass - if tree in ("genexpr_func", "genexpr_func_async",): + if tree in ( + "genexpr_func", + "genexpr_func_async", + ): for i in range(3, 5): if tree[i] == "comp_iter": iter_index = i @@ -330,8 +333,19 @@ class ComprehensionMixin: assert store == "store" n = set_iter_async[2] elif node == "list_comp" and tree[0] == "expr": - tree = tree[0][0] - n = tree[iter_index] + list_iter = None + for list_iter_try in tree: + if list_iter_try == "list_iter": + list_iter = list_iter_try + break + if not list_iter_try: + tree = tree[0][0] + n = tree[iter_index] + else: + n = list_iter + pass + pass + pass else: n = tree[iter_index] @@ -405,6 +419,9 @@ class ComprehensionMixin: n = n[0] if n in ("list_for", "comp_for"): + if n == "list_for" and not comp_for and n[0] == "expr": + comp_for = n[0] + n_index = 3 if ( (n[2] == "store") @@ -494,11 +511,21 @@ class ComprehensionMixin: if comp_for: self.preorder(comp_for) else: + try: + node[in_node_index] + except: + from trepan.api import debug + + debug() self.preorder(node[in_node_index]) # Here is where we handle nested list iterations. if tree == "list_comp" and self.version != (3, 0): - list_iter = tree[1] + list_iter = None + for list_iter_try in tree: + if list_iter_try == "list_iter": + list_iter = list_iter_try + break assert list_iter == "list_iter" if list_iter[0] == "list_for": self.preorder(list_iter[0][3]) @@ -637,7 +664,6 @@ class ComprehensionMixin: # Find the list comprehension body. It is the inner-most # node that is not list_.. . while n == "list_iter": - # recurse one step n = n[0] diff --git a/uncompyle6/semantics/n_actions.py b/uncompyle6/semantics/n_actions.py index cc520ac2..d48ea3fb 100644 --- a/uncompyle6/semantics/n_actions.py +++ b/uncompyle6/semantics/n_actions.py @@ -1036,7 +1036,7 @@ class NonterminalActions: self.prec = p self.prune() # stop recursing - def n_listcomp(self, node): + def n_list_comp(self, node): self.write("[") if node[0].kind == "load_closure": assert self.version >= (3, 0)