From b84c89e81780c40ee74ef17d87302b874cebe4c2 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 11 Jan 2020 22:36:07 -0500 Subject: [PATCH] Bang on getting 3.x "try" vs "try/else" disambiguated --- test/stdlib/runtests.sh | 4 +- uncompyle6/parsers/parse3.py | 70 ++++++++++++------- uncompyle6/parsers/parse37base.py | 2 + uncompyle6/parsers/parse38.py | 1 + uncompyle6/parsers/reducecheck/__init__.py | 1 + uncompyle6/parsers/reducecheck/tryelsestmt.py | 3 +- .../parsers/reducecheck/tryelsestmtl3.py | 41 +++++++++++ 7 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 uncompyle6/parsers/reducecheck/tryelsestmtl3.py diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index a767dd75..aebf83d0 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -207,7 +207,6 @@ case $PYVERSION in [test_dis.py]=1 # We change line numbers - duh! [test_exceptions.py]=1 # [test_faulthandler.py]=1 - [test_fileio.py]=1 [test_grammar.py]=1 [test_lib2to3.py]=1 [test_math.py]=1 @@ -278,7 +277,6 @@ case $PYVERSION in [test_enum.py]=1 [test_exceptions.py]=1 [test_faulthandler.py]=1 - [test_fileio.py]=1 [test_file_eintr.py]=1 # parse error [test_fileinput.py]=1 # doesn't terminate [test_fork1.py]=1 # too long @@ -406,7 +404,6 @@ case $PYVERSION in [test_enum.py]=1 # [test_exceptions.py]=1 # parse error [test_fileinput.py]=1 # doesn't terminate - [test_fileio.py]=1 [test_fractions.py]=1 # doesn't terminate [test_frame.py]=1 # doesn't terminate [test_ftplib.py]=1 # doesn't terminate @@ -638,6 +635,7 @@ case $PYVERSION in [test_decorators.py]=1 # Control flow wrt "if elif" [test_exceptions.py]=1 # parse error [test_dis.py]=1 # We change line numbers - duh! + [test_fileio.py]=1 # parse error [test_pow.py]=1 # Control flow wrt "continue" [test_quopri.py]=1 # Only fails on POWER # ... diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 2210dd2f..d1d3e4dc 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -29,7 +29,7 @@ that a later phase can turn into a sequence of ASCII text. import re from uncompyle6.scanners.tok import Token from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func -from uncompyle6.parsers.reducecheck import except_handler_else, testtrue +from uncompyle6.parsers.reducecheck import except_handler_else, testtrue, tryelsestmtl3 from uncompyle6.parsers.treenode import SyntaxTree from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from xdis import PYTHON3 @@ -1015,7 +1015,9 @@ class Python3Parser(PythonParser): self.add_make_function_rule(rule_pat, opname, token.attr, customize) if has_get_iter_call_function1: - if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"): + if self.is_pypy or ( + i >= j and tokens[i - j] == "LOAD_LISTCOMP" + ): # In the tokens we saw: # LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or # LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or @@ -1038,7 +1040,9 @@ class Python3Parser(PythonParser): self.add_make_function_rule( rule_pat, opname, token.attr, customize ) - if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_DICTCOMP"): + if self.is_pypy or ( + i >= j and tokens[i - j] == "LOAD_DICTCOMP" + ): self.add_unique_rule( "dict_comp ::= %sload_closure LOAD_DICTCOMP %s " "expr GET_ITER CALL_FUNCTION_1" @@ -1056,11 +1060,14 @@ class Python3Parser(PythonParser): # Note order of kwargs and pos args changed between 3.3-3.4 if self.version <= 3.2: if annotate_args > 0: - rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE %s" % ( - kwargs_str, - "pos_arg " * args_pos, - "annotate_arg " * (annotate_args - 1), - opname, + rule = ( + "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE %s" + % ( + kwargs_str, + "pos_arg " * args_pos, + "annotate_arg " * (annotate_args - 1), + opname, + ) ) else: rule = "mkfunc ::= %s%sload_closure LOAD_CODE %s" % ( @@ -1070,11 +1077,14 @@ class Python3Parser(PythonParser): ) elif self.version == 3.3: if annotate_args > 0: - rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" % ( - kwargs_str, - "pos_arg " * args_pos, - "annotate_arg " * (annotate_args - 1), - opname, + rule = ( + "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" + % ( + kwargs_str, + "pos_arg " * args_pos, + "annotate_arg " * (annotate_args - 1), + opname, + ) ) else: rule = "mkfunc ::= %s%sload_closure LOAD_CODE LOAD_STR %s" % ( @@ -1090,12 +1100,15 @@ class Python3Parser(PythonParser): load_op = "LOAD_CONST" if annotate_args > 0: - rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure %s %s" % ( - "pos_arg " * args_pos, - kwargs_str, - "annotate_arg " * (annotate_args - 1), - load_op, - opname, + rule = ( + "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure %s %s" + % ( + "pos_arg " * args_pos, + kwargs_str, + "annotate_arg " * (annotate_args - 1), + load_op, + opname, + ) ) else: rule = "mkfunc ::= %s%s load_closure LOAD_CODE %s %s" % ( @@ -1183,7 +1196,9 @@ class Python3Parser(PythonParser): self.add_make_function_rule( rule_pat, opname, token.attr, customize ) - if self.is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP"): + if self.is_pypy or ( + i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP" + ): if self.version >= 3.6: # 3.6+ sometimes bundles all of the # 'exprs' in the rule above into a @@ -1437,7 +1452,7 @@ class Python3Parser(PythonParser): tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK except_handler_else else_suite come_froms """, - nop_func + nop_func, ) custom_ops_processed.add(opname) @@ -1472,6 +1487,7 @@ class Python3Parser(PythonParser): # 3.6+ can remove a JUMP_FORWARD which messes up our testing here self.check_reduce["try_except"] = "AST" + self.check_reduce["tryelsestmtl3"] = "AST" # FIXME: remove parser errors caused by the below # self.check_reduce['while1elsestmt'] = 'noAST' return @@ -1491,7 +1507,7 @@ class Python3Parser(PythonParser): elif rule == ("ifstmt", ("testexpr", "_ifstmts_jump")): condition_jump = ast[0].last_child() if condition_jump.kind.startswith("POP_JUMP_IF"): - condition_jump2 = tokens[min(last-1, len(tokens)-1)] + condition_jump2 = tokens[min(last - 1, len(tokens) - 1)] if condition_jump2.kind.startswith("POP_JUMP_IF"): return condition_jump.attr == condition_jump2.attr # if condition_jump.attr < condition_jump2.off2int(): @@ -1501,13 +1517,19 @@ class Python3Parser(PythonParser): return condition_jump.attr < condition_jump2.off2int() return False elif lhs == "ifelsestmt" and rule[1][2] == "jump_forward_else": - last = min(last, len(tokens)-1) + last = min(last, len(tokens) - 1) if tokens[last].off2int() == -1: last -= 1 jump_forward_else = ast[2] - return tokens[first].off2int() <= jump_forward_else[0].attr < tokens[last].off2int() + return ( + tokens[first].off2int() + <= jump_forward_else[0].attr + < tokens[last].off2int() + ) elif lhs == "testtrue": return testtrue(self, lhs, n, rule, ast, tokens, first, last) + elif lhs == "tryelsestmtl3": + return tryelsestmtl3(self, lhs, n, rule, ast, tokens, first, last) elif lhs == "while1stmt": # If there is a fall through to the COME_FROM_LOOP, then this is diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index 00e6c362..c5c30049 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -15,6 +15,7 @@ from uncompyle6.parsers.reducecheck import ( ifstmts_jump, or_check, testtrue, + tryelsestmtl3, while1stmt, while1elsestmt, ) @@ -1000,6 +1001,7 @@ class Python37BaseParser(PythonParser): "testtrue": testtrue, "while1elsestmt": while1elsestmt, "while1stmt": while1stmt, + "try_elsestmtl38": tryelsestmtl3, } self.check_reduce["and"] = "AST" diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index e84efd96..7a481386 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -241,6 +241,7 @@ class Python38Parser(Python37Parser): self.remove_rules_38() self.check_reduce["whileTruestmt38"] = "tokens" self.check_reduce["whilestmt38"] = "tokens" + self.check_reduce["try_elsestmtl38"] = "AST" def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python38Parser, diff --git a/uncompyle6/parsers/reducecheck/__init__.py b/uncompyle6/parsers/reducecheck/__init__.py index edd374e1..daa842c3 100644 --- a/uncompyle6/parsers/reducecheck/__init__.py +++ b/uncompyle6/parsers/reducecheck/__init__.py @@ -7,5 +7,6 @@ from uncompyle6.parsers.reducecheck.ifstmts_jump import * from uncompyle6.parsers.reducecheck.or_check import * from uncompyle6.parsers.reducecheck.testtrue import * from uncompyle6.parsers.reducecheck.tryelsestmt import * +from uncompyle6.parsers.reducecheck.tryelsestmtl3 import * from uncompyle6.parsers.reducecheck.while1elsestmt import * from uncompyle6.parsers.reducecheck.while1stmt import * diff --git a/uncompyle6/parsers/reducecheck/tryelsestmt.py b/uncompyle6/parsers/reducecheck/tryelsestmt.py index bf8625e0..cffa51b2 100644 --- a/uncompyle6/parsers/reducecheck/tryelsestmt.py +++ b/uncompyle6/parsers/reducecheck/tryelsestmt.py @@ -6,13 +6,14 @@ def tryelsestmt(self, lhs, n, rule, ast, tokens, first, last): # Check the end of the except handler that there isn't a jump from # inside the except handler to the end. If that happens # then this is a "try" with no "else". + from trepan.api import debug; debug() except_handler = ast[3] if except_handler == "except_handler": come_from = except_handler[-1] # We only care about the *first* come_from because that is the # the innermost one. So if the "tryelse" is invalid (should be a "try") - # ti will be invalid here. + # it will be invalid here. if come_from == "COME_FROM": first_come_from = except_handler[-1] elif come_from == "END_FINALLY": diff --git a/uncompyle6/parsers/reducecheck/tryelsestmtl3.py b/uncompyle6/parsers/reducecheck/tryelsestmtl3.py new file mode 100644 index 00000000..acd0b9b8 --- /dev/null +++ b/uncompyle6/parsers/reducecheck/tryelsestmtl3.py @@ -0,0 +1,41 @@ +# Copyright (c) 2020 Rocky Bernstein + +from uncompyle6.parsers.treenode import SyntaxTree + +def tryelsestmtl3(self, lhs, n, rule, ast, tokens, first, last): + # Check the end of the except handler that there isn't a jump from + # inside the except handler to the end. If that happens + # then this is a "try" with no "else". + except_handler = ast[3] + + if except_handler == "except_handler_else": + except_handler = except_handler[0] + + come_from = except_handler[-1] + # We only care about the *first* come_from because that is the + # the innermost one. So if the "tryelse" is invalid (should be a "try") + # it will be invalid here. + if come_from == "COME_FROM": + first_come_from = except_handler[-1] + elif come_from == "END_FINALLY": + return False + elif come_from == "except_return": + return False + else: + assert come_from in ("come_froms", "opt_come_from_except") + first_come_from = come_from[0] + if not hasattr(first_come_from, "attr"): + # optional come from + return False + + leading_jump = except_handler[0] + + # We really don't care that this is a jump per-se. But + # we could also check that this jumps to the end of the except if + # desired. + if isinstance(leading_jump, SyntaxTree): + except_handler_first_offset = leading_jump.first_child().off2int() + else: + except_handler_first_offset = leading_jump.off2int() + + return first_come_from.attr > except_handler_first_offset