From 05f3dad32cb32183b12af21c5d2f225d20264649 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 11 Dec 2019 06:56:43 -0500 Subject: [PATCH] Fix some 3.7+ "if"/"and" logic bugs --- ...st_pysource.py => test_pysource.py-notyet} | 0 uncompyle6/parsers/parse37.py | 2 + uncompyle6/parsers/parse37base.py | 85 +++++++++++++------ uncompyle6/parsers/parse38.py | 27 +----- uncompyle6/scanners/scanner37base.py | 19 ----- uncompyle6/semantics/customize38.py | 2 +- uncompyle6/semantics/transform.py | 6 +- 7 files changed, 67 insertions(+), 74 deletions(-) rename pytest/{test_pysource.py => test_pysource.py-notyet} (100%) diff --git a/pytest/test_pysource.py b/pytest/test_pysource.py-notyet similarity index 100% rename from pytest/test_pysource.py rename to pytest/test_pysource.py-notyet diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index 2e2d676c..bd28bfd4 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -950,6 +950,8 @@ class Python37Parser(Python37BaseParser): ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond + jitop_come_from ::= JUMP_IF_TRUE_OR_POP COME_FROM + or ::= and jitop_come_from expr COME_FROM or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM or ::= expr JUMP_IF_TRUE expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index d604430c..48856279 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -936,6 +936,7 @@ class Python37BaseParser(PythonParser): self.check_reduce["iflaststmt"] = "AST" self.check_reduce["iflaststmtl"] = "AST" self.check_reduce["ifstmt"] = "AST" + self.check_reduce["ifstmtl"] = "AST" self.check_reduce["annotate_tuple"] = "noAST" # FIXME: remove parser errors caused by the below @@ -1028,7 +1029,6 @@ class Python37BaseParser(PythonParser): if lhs == "and" and ast: # FIXME: put in a routine somewhere - # Compare with parse30.py of uncompyle6 jmp = ast[1] if jmp.kind.startswith("jmp_"): if last == n: @@ -1043,7 +1043,11 @@ class Python37BaseParser(PythonParser): return jmp_target != jmp2_target elif rule == ("and", ("expr", "jmp_false", "expr")): if tokens[last] == "POP_JUMP_IF_FALSE": + # Ok if jump it doesn't jump to last instruction return jmp_target != tokens[last].attr + elif tokens[last] == "JUMP_IF_TRUE_OR_POP": + # Ok if jump it does jump right after last instruction + return jmp_target + 2 != tokens[last].attr elif rule == ("and", ("expr", "jmp_false", "expr", "COME_FROM")): return ast[-1].attr != jmp_offset # elif rule == ("and", ("expr", "jmp_false", "expr", "COME_FROM")): @@ -1153,36 +1157,65 @@ class Python37BaseParser(PythonParser): else: return tokens[pop_jump_index].offset > come_froms[-1].attr - elif lhs == "ifstmt" and ast: + elif lhs in ("ifstmt", "ifstmtl"): # FIXME: put in a routine somewhere - testexpr = ast[0] - if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP": - # iflastsmtl jumped outside of loop. No good. - return True + n = len(tokens) + if lhs == "ifstmtl": + if last == n: + last -= 1 + pass + if (tokens[last].attr and isinstance(tokens[last].attr, int)): + return tokens[first].offset < tokens[last].attr + pass - if testexpr[0] in ("testtrue", "testfalse"): - test = testexpr[0] - if len(test) > 1 and test[1].kind.startswith("jmp_"): - if last == n: - last -= 1 - jmp_target = test[1][0].attr - if tokens[first].off2int() <= jmp_target < tokens[last].off2int(): - return True - # jmp_target less than tokens[first] is okay - is to a loop - # jmp_target equal tokens[last] is also okay: normal non-optimized non-loop jump - if jmp_target > tokens[last].off2int(): - # One more weird case to look out for - # if c1: - # if c2: # Jumps around the *outer* "else" - # ... - # else: - if jmp_target == tokens[last - 1].attr: - return False - if last < n and tokens[last].kind.startswith("JUMP"): - return False + # Make sure jumps don't extend beyond the end of the if statement. + l = last + if l == n: + l -= 1 + if isinstance(tokens[l].offset, str): + last_offset = int(tokens[l].offset.split("_")[0], 10) + else: + last_offset = tokens[l].offset + for i in range(first, l): + t = tokens[i] + if t.kind == "POP_JUMP_IF_FALSE": + if t.attr > last_offset: return True + pass + pass + pass + if ast: + testexpr = ast[0] + + if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP": + # iflastsmtl jumped outside of loop. No good. + return True + + if testexpr[0] in ("testtrue", "testfalse"): + test = testexpr[0] + if len(test) > 1 and test[1].kind.startswith("jmp_"): + if last == n: + last -= 1 + jmp_target = test[1][0].attr + if tokens[first].off2int() <= jmp_target < tokens[last].off2int(): + return True + # jmp_target less than tokens[first] is okay - is to a loop + # jmp_target equal tokens[last] is also okay: normal non-optimized non-loop jump + if jmp_target > tokens[last].off2int(): + # One more weird case to look out for + # if c1: + # if c2: # Jumps around the *outer* "else" + # ... + # else: + if jmp_target == tokens[last - 1].attr: + return False + if last < n and tokens[last].kind.startswith("JUMP"): + return False + return True + + pass pass return False elif lhs in ("iflaststmt", "iflaststmtl") and ast: diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index c82120c8..35fe2b26 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -263,10 +263,8 @@ class Python38Parser(Python37Parser): def customize_grammar_rules(self, tokens, customize): super(Python37Parser, self).customize_grammar_rules(tokens, customize) self.remove_rules_38() - self.check_reduce["ifstmt"] = "tokens" self.check_reduce["whileTruestmt38"] = "tokens" self.check_reduce["whilestmt38"] = "tokens" - self.check_reduce["ifstmtl"] = "tokens" def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python38Parser, @@ -276,30 +274,7 @@ class Python38Parser(Python37Parser): if invalid: return invalid lhs = rule[0] - if lhs == "ifstmt": - # Make sure jumps don't extend beyond the end of the if statement. - l = last - if l == len(tokens): - l -= 1 - if isinstance(tokens[l].offset, str): - last_offset = int(tokens[l].offset.split("_")[0], 10) - else: - last_offset = tokens[l].offset - for i in range(first, l): - t = tokens[i] - if t.kind == "POP_JUMP_IF_FALSE": - if t.attr > last_offset: - return True - pass - pass - pass - elif lhs == "ifstmtl": - if last == len(tokens): - last -= 1 - if (tokens[last].attr and isinstance(tokens[last].attr, int)): - return tokens[first].offset < tokens[last].attr - pass - elif lhs in ("whileTruestmt38", "whilestmt38"): + if lhs in ("whileTruestmt38", "whilestmt38"): jb_index = last - 1 while jb_index > 0 and tokens[jb_index].kind.startswith("COME_FROM"): jb_index -= 1 diff --git a/uncompyle6/scanners/scanner37base.py b/uncompyle6/scanners/scanner37base.py index 9519724a..8f7fd7e0 100644 --- a/uncompyle6/scanners/scanner37base.py +++ b/uncompyle6/scanners/scanner37base.py @@ -776,25 +776,6 @@ class Scanner37Base(Scanner): ) elif op in self.pop_jump_tf: target = inst.argval - prev_op = self.prev_op - - # FIXME: hack upon hack, test_pysource.py fails with this - # Until the grammar is corrected we do this fiction... - pretarget = self.get_inst(prev_op[target]) - if ( - pretarget.opcode in self.pop_jump_if_pop - and (target > offset) - and pretarget.offset != offset - ): - - if pretarget.argval != target: - # FIXME: this is not accurate The commented out below - # is what it should be. However grammar rules right now - # assume the incorrect offsets. - # self.fixed_jumps[offset] = target - self.fixed_jumps[offset] = pretarget.offset - return - self.fixed_jumps[offset] = target elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT: diff --git a/uncompyle6/semantics/customize38.py b/uncompyle6/semantics/customize38.py index 9795088e..8e9c15b0 100644 --- a/uncompyle6/semantics/customize38.py +++ b/uncompyle6/semantics/customize38.py @@ -111,7 +111,7 @@ def customize_for_version38(self, version): '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', (1, "suite_stmts_opt"), (6, "suite_stmts_opt") ), - 'tryfinally38_return_stmt': ( + 'tryfinally38astmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', (2, "suite_stmts_opt"), (8, "suite_stmts_opt") ), diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index d3f88bba..adbac4aa 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -96,7 +96,8 @@ class TreeTransform(GenericASTTraversal, object): jump_cond = testexpr[0][1] expr = raise_stmt[0] RAISE_VARARGS_1 = raise_stmt[1] - if expr[0] == "call": + call = expr[0] + if call == "call": # ifstmt # 0. testexpr # testtrue (2) @@ -116,7 +117,8 @@ class TreeTransform(GenericASTTraversal, object): assert jump_cond == "jmp_false" kind = "assert2not" - call = expr[0] + if call[0] != "LOAD_ASSERT": + return node LOAD_ASSERT = call[0] if isinstance(call[1], SyntaxTree): expr = call[1][0]