From c93a7a728b199b0db0f05da6848b28474dd2add0 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 7 Feb 2020 18:29:05 -0500 Subject: [PATCH] Add decompyle3 ifelsestmt reduction rule... and Go over 3.3 and 3.7 runtests excludes --- test/stdlib/3.3-exclude.sh | 12 -- test/stdlib/3.7-exclude.sh | 13 ++ uncompyle6/parsers/parse3.py | 3 + uncompyle6/parsers/reducecheck/ifelsestmt.py | 136 +++++++++++-------- 4 files changed, 98 insertions(+), 66 deletions(-) diff --git a/test/stdlib/3.3-exclude.sh b/test/stdlib/3.3-exclude.sh index 3c436195..3ffc9191 100644 --- a/test/stdlib/3.3-exclude.sh +++ b/test/stdlib/3.3-exclude.sh @@ -1,16 +1,4 @@ SKIP_TESTS=( - [test_binop.py]=1 # FIXME: Works on c90ff51 - [test_cgi.py]=1 # FIXME: Works on c90ff51 - [test_decorators.py]=1 # FIXME: Works on c90ff51 - [test_pyclbr.py]=1 # FIXME: Works on c90ff51 - [test_optparse.py]=1 # FIXME: Works on c90ff51 - [test_os.py]=1 # FIXME: Works on c90ff51 - [test_pep352.py]=1 # FIXME: Works on c90ff51 - [test_pyclbr.py]=1 # FIXME: Works on c90ff51 - [test_shutil.py]=1 # FIXME: Works on c90ff51 - [test_strftime.py]=1 # FIXME: Works on c90ff51 - [test_symtable.py]=1 # FIXME: Works on c90ff51 - [test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines [test_buffer.py]=1 # parse error diff --git a/test/stdlib/3.7-exclude.sh b/test/stdlib/3.7-exclude.sh index cafd9012..9f8a131f 100644 --- a/test/stdlib/3.7-exclude.sh +++ b/test/stdlib/3.7-exclude.sh @@ -1,4 +1,17 @@ SKIP_TESTS=( + [test_builtin.py]=1 # FIXME works on decompyle6 + [test_context.py]=1 # FIXME works on decompyle6 + [test_format.py]=1 # FIXME works on decompyle6 + [test_marshal.py]=1 # FIXME works on decompyle6 + [test_normalization.py]=1 # FIXME works on decompyle6 + [test_os.py]=1 # FIXME works on decompyle6 + [test_slice.py]=1 # FIXME works on decompyle6 + [test_sort.py]=1 # FIXME works on decompyle6 + [test_statistics.py]=1 # FIXME works on decompyle6 + [test_string_literals.py]=1 # FIXME works on decompyle6 + [test_timeit.py]=1 # FIXME works on decompyle6 + [test_urllib_localnet.py]=1 # FIXME works on decompyle6 + [test_urllib2.py]=1 # FIXME: works on uncompyle6 [test_generators.py]=1 # Investigate improper lamdba with bogus "False" added [test_grammar.py]=1 # investigate: like above: semantic rule missing probably diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 2d601485..892b2d1c 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -32,6 +32,7 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parsers.reducecheck import ( and_check, except_handler_else, + ifelsestmt, ifstmt, iflaststmt, testtrue, @@ -1531,6 +1532,7 @@ class Python3Parser(PythonParser): self.reduce_check_table = { "except_handler_else": except_handler_else, # "ifstmt": ifstmt, + "ifelsestmtc": ifstmt, "testtrue": testtrue, "tryelsestmtl3": tryelsestmtl3, "try_except": tryexcept, @@ -1545,6 +1547,7 @@ class Python3Parser(PythonParser): self.check_reduce["while1stmt"] = "noAST" self.check_reduce["while1elsestmt"] = "noAST" self.check_reduce["ifelsestmt"] = "AST" + self.check_reduce["ifelsestmtc"] = "AST" self.check_reduce["ifstmt"] = "AST" if self.version == 3.6: self.reduce_check_table["iflaststmtl"] = iflaststmt diff --git a/uncompyle6/parsers/reducecheck/ifelsestmt.py b/uncompyle6/parsers/reducecheck/ifelsestmt.py index 7583c2ea..1eeabe21 100644 --- a/uncompyle6/parsers/reducecheck/ifelsestmt.py +++ b/uncompyle6/parsers/reducecheck/ifelsestmt.py @@ -2,14 +2,9 @@ from uncompyle6.scanners.tok import Token - -def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): - if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP": - # ifelsestmt jumped outside of loop. No good. - return True - - if rule not in ( - ( +IFELSE_STMT_RULES = frozenset( + [ + ( "ifelsestmt", ( "testexpr", @@ -29,6 +24,16 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): "\\e__come_froms", ), ), + ( + "ifelsestmtc", + ( + "testexpr", + "c_stmts_opt", + "jump_forward_else", + "else_suitec", + "\\e__come_froms", + ), + ), ( "ifelsestmt", ( @@ -72,84 +77,107 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): "else_suite", ), ), - ): + ]) + +def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): + if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP": + # ifelsestmt jumped outside of loop. No good. + return True + + if rule not in IFELSE_STMT_RULES: return False + # Make sure all of the "come froms" offset at the # end of the "if" come from somewhere inside the "if". # Since the come_froms are ordered so that lowest # offset COME_FROM is last, it is sufficient to test # just the last one. - come_froms = ast[-1] - if come_froms.kind != "else_suite" and self.version >= 3.0: - if come_froms == "opt_come_from_except" and len(come_froms) > 0: - come_froms = come_froms[0] - if not isinstance(come_froms, Token): - if len(come_froms): - return tokens[first].offset > come_froms[-1].attr - elif tokens[first].offset > come_froms.attr: - return True + if len(ast) == 5: + end_come_froms = ast[-1] + if end_come_froms.kind != "else_suite" and self.version >= 3.0: + if end_come_froms == "opt_come_from_except" and len(end_come_froms) > 0: + end_come_froms = end_come_froms[0] + if not isinstance(end_come_froms, Token): + if len(end_come_froms): + return tokens[first].offset > end_come_froms[-1].attr + elif tokens[first].offset > end_come_froms.attr: + return True - # FIXME: There is weirdness in the grammar we need to work around. - # we need to clean up the grammar. - if self.version < 3.0: - last_token = ast[-1] - else: - last_token = tokens[last] - if last_token == "COME_FROM" and tokens[first].offset > last_token.attr: - if self.version < 3.0 and self.insts[self.offset2inst_index[last_token.attr]].opname != "SETUP_LOOP": - return True + # FIXME: There is weirdness in the grammar we need to work around. + # we need to clean up the grammar. + if self.version < 3.0: + last_token = ast[-1] + else: + last_token = tokens[last] + if last_token == "COME_FROM" and tokens[first].offset > last_token.attr: + if self.version < 3.0 and self.insts[self.offset2inst_index[last_token.attr]].opname != "SETUP_LOOP": + return True testexpr = ast[0] # Check that the condition portion of the "if" # jumps to the "else" part. if testexpr[0] in ("testtrue", "testfalse"): - test = testexpr[0] + if_condition = testexpr[0] else_suite = ast[3] - assert else_suite == "else_suite" + assert else_suite.kind.startswith("else_suite") - if len(test) > 1 and test[1].kind.startswith("jmp_"): + if len(if_condition) > 1 and if_condition[1].kind.startswith("jmp_"): if last == n: last -= 1 - jmp = test[1] + jmp = if_condition[1] if self.version > 2.6: jmp_target = jmp[0].attr else: jmp_target = int(jmp[0].pattr) - # Make sure we don't jump at the end of the "then" inside the "else" - # (jf_cf_pop may be a 2.6ish specific thing.) - jump_forward = ast[2] - if jump_forward == "jf_cf_pop": - jump_forward = jump_forward[0] + # Below we check that jmp_target is jumping to a feasible + # location. It should be to the transition after the "then" + # block and to the beginning of the "else" block. + # However the "if/else" is inside a loop the false test can be + # back to the loop. + + # FIXME: the below logic for jf_cfs could probably be + # simplified. + jump_else_end = ast[2] + if jump_else_end == "jf_cf_pop": + jump_else_end = jump_else_end[0] jump_to_jump = False - if jump_forward == "JUMP_FORWARD": + if jump_else_end == "JUMP_FORWARD": jump_to_jump = True - endif_target = int(jump_forward.pattr) + endif_target = int(jump_else_end.pattr) last_offset = tokens[last].off2int() if endif_target != last_offset: return True + last_offset = tokens[last].off2int(prefer_last=False) + if jmp_target == last_offset: + # jmp_target should be jumping to the end of the if/then/else + # but is it jumping to the beginning of the "else" + return True + if ( + jump_else_end in ("jf_cfs", "jump_forward_else") + and jump_else_end[0] == "JUMP_FORWARD" + ): + # If the "else" jump jumps before the end of the the "if .. else end", then this + # is not this kind of "ifelsestmt". + jump_else_forward = jump_else_end[0] + jump_else_forward_target = jump_else_forward.attr + if jump_else_forward_target < last_offset: + return True + pass + if ( + jump_else_end in ("jb_elsec", "jb_elsel", "jf_cfs", "jb_cfs") + and jump_else_end[-1] == "COME_FROM" + ): + if jump_else_end[-1].off2int() != jmp_target: + return True - # The jump inside "else" check below should be added. - # add this until we can find out what's wrong with - # not being able to parse: - # if a and b or c: - # x = 1 - # else: - # x = 2 - - # FIXME: add this - # if jmp_target < else_suite.first_child().off2int(): - # return True - - if tokens[first].off2int() > jmp_target and not jump_to_jump: + if tokens[first].off2int() > jmp_target: return True - return (jmp_target > tokens[last].off2int()) and tokens[ - last - ] != "JUMP_FORWARD" + return (jmp_target > last_offset) and tokens[last] != "JUMP_FORWARD" return False