diff --git a/Makefile b/Makefile index 888e1df8..363b1dd4 100644 --- a/Makefile +++ b/Makefile @@ -48,10 +48,14 @@ check-2.4 check-2.5: # Skip for now 2.6 5.0 5.3 5.6 5.8: -#:PyPy pypy3-2.4.0 Python 3: +#:PyPy pypy3-2.4.0 Python 3.6.1: 7.1 pypy-3.2 2.4: $(MAKE) -C test $@ +#:PyPy pypy3-2.4.0 Python 3.6.9: +7.2: + $(MAKE) -C test $@ + #: Run py.test tests pytest: $(MAKE) -C pytest check diff --git a/test/Makefile b/test/Makefile index 22ada93b..680a694d 100644 --- a/test/Makefile +++ b/test/Makefile @@ -337,12 +337,18 @@ pypy-2.7 5.0 5.3 6.0: pypy-3.2 2.4: $(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify -#: PyPy 5.0.x with Python 3.6 ... +#: PyPy 5.0.x with Python 3.6.1 ... check-bytecode-pypy3.6: 7.1 7.1: $(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify +#: PyPy 5.0.x with Python 3.6.9 +check-bytecode-pypy3.6: 7.2 +7.2: + $(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify + clean: clean-py-dis clean-dis clean-unverified diff --git a/test/bytecode_3.0/02_try_except_except.pyc b/test/bytecode_3.0/02_try_except_except.pyc index 2d655465..a56e0021 100644 Binary files a/test/bytecode_3.0/02_try_except_except.pyc and b/test/bytecode_3.0/02_try_except_except.pyc differ diff --git a/test/bytecode_3.0/02_while1_if_while1.pyc b/test/bytecode_3.0/02_while1_if_while1.pyc index c4282d8e..7e44fe49 100644 Binary files a/test/bytecode_3.0/02_while1_if_while1.pyc and b/test/bytecode_3.0/02_while1_if_while1.pyc differ diff --git a/test/bytecode_3.0/03_ifelse.pyc b/test/bytecode_3.0/03_ifelse.pyc new file mode 100644 index 00000000..aa09846c Binary files /dev/null and b/test/bytecode_3.0/03_ifelse.pyc differ diff --git a/test/bytecode_3.0_run/04_and_del.pyc b/test/bytecode_3.0_run/04_and_del.pyc index 06a2ddeb..6d535339 100644 Binary files a/test/bytecode_3.0_run/04_and_del.pyc and b/test/bytecode_3.0_run/04_and_del.pyc differ diff --git a/test/simple_source/bug30/02_try_except_except.py b/test/simple_source/bug30/02_try_except_except.py index 56c729a9..3bbc94e7 100644 --- a/test/simple_source/bug30/02_try_except_except.py +++ b/test/simple_source/bug30/02_try_except_except.py @@ -7,3 +7,13 @@ def start_new_thread(function, args, kwargs={}): pass except: args() + +# Adapted from 3.0.1 code.py +# Bug is again JUMP_FORWARD elimination compared +# to earlier and later Pythons. +def interact(): + while 1: + try: + more = 1 + except KeyboardInterrupt: + more = 0 diff --git a/test/simple_source/bug30/02_while1_if_while1.py b/test/simple_source/bug30/02_while1_if_while1.py index 849bf110..f7ba63a5 100644 --- a/test/simple_source/bug30/02_while1_if_while1.py +++ b/test/simple_source/bug30/02_while1_if_while1.py @@ -7,3 +7,8 @@ while 1: raise RuntimeError else: raise RuntimeError + +# Adapted from 3.0.1 cgi.py +def _parseparam(s, end): + while end > 0 and s.count(''): + end = s.find(';') diff --git a/test/simple_source/bug30/03_ifelse.py b/test/simple_source/bug30/03_ifelse.py new file mode 100644 index 00000000..0f36b953 --- /dev/null +++ b/test/simple_source/bug30/03_ifelse.py @@ -0,0 +1,50 @@ +# Adapted from 3.0 base64 +# Problem was handling if/else which +# needs to be like Python 2.6 (and not like 2.7 or 3.1) +def main(args, f, func, sys): + """Small main program""" + if args and args[0] != '-': + func(f, sys.stdout.buffer) + else: + func(sys.stdin.buffer, sys.stdout.buffer) + +# From Python 3.0 _markupbase.py. +# +# The Problem was in the way "if"s are generated in 3.0 which is sort +# of like a more optimized Python 2.6, with reduced extraneous jumps, +# but still 2.6-ish and not 2.7- or 3.1-ish. +def parse_marked_section(fn, i, rawdata, report=1): + if report: + j = 1 + fn(rawdata[i: j]) + return 10 + +# From 3.0.1 _abcoll.py +# Bug was in genexpr_func which doesn't have a JUMP_BACK but +# in its gen_comp_body, we can use COME_FROM in its place. +# As above omission of JUMPs is a feature of 3.0 that doesn't +# seem to be in later versions (or earlier like 2.6). +def __and__(self, other, Iterable): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + +# Adapted from 3.0.1 abc.py +# Bug was in handling multiple COME_FROMs in return_if_stmt +def __instancecheck__(subtype, subclass, cls): + if subtype: + if (cls and subclass): + return False + + +# Adapted from 3.0.1 abc.py +# Bug was rule in "jump_absolute_else" and disallowing +# "else" to the wrong place. + +def _strptime(locale_time, found_zone, time): + for tz_values in locale_time: + if found_zone: + if (time and found_zone): + break + else: + break diff --git a/test/simple_source/bug30/04_and_del.py b/test/simple_source/bug30/04_and_del.py index acc13be6..51b452e6 100644 --- a/test/simple_source/bug30/04_and_del.py +++ b/test/simple_source/bug30/04_and_del.py @@ -18,3 +18,13 @@ assert normpath(['.']) == [] assert normpath(['a', 'b', '..']) == ['a'] assert normpath(['a', 'b', '', 'c']) == ['a', 'b', 'c'] assert normpath(['a', 'b', '.', '', 'c', '..']) == ['a', 'b'] + +# Adapted from 3.0.1 cgi.py +# Bug was in detecting "or" and "and" due to lack of PUSH/POP_IF instructions. +def handle(format, html, text): + formatter = (format and html) or text + return formatter + +assert handle(False, False, True) +assert not handle(True, False, False) +assert handle(True, True, False) diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index d01e3f71..24bc236a 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -43,6 +43,8 @@ TEST_VERSIONS = ( "pypy3.5-5.9.0", "pypy3.5-6.0.0", "pypy3.6-7.1.0", + "pypy3.6-7.1.1", + "pypy3.6-7.2.0", "native", ) + tuple(python_versions) diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index 33012d0d..2d91e9ad 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -254,7 +254,7 @@ class Python26Parser(Python2Parser): POP_TOP jb_pb_come_from generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM - list_if ::= list_if ::= expr jmp_false_then list_iter + list_if ::= expr jmp_false_then list_iter ''' def p_ret26(self, args): @@ -467,7 +467,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 2.6: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse27.py b/uncompyle6/parsers/parse27.py index fc124004..18b1b314 100644 --- a/uncompyle6/parsers/parse27.py +++ b/uncompyle6/parsers/parse27.py @@ -292,7 +292,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 2.7: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index c29ca748..7f528f01 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1439,8 +1439,9 @@ class Python3Parser(PythonParser): except_handler COME_FROM else_suitel opt_come_from_except """, - nop_func, + nop_func ) + custom_ops_processed.add(opname) elif opname_base in ("UNPACK_EX",): before_count, after_count = token.attr @@ -1543,12 +1544,24 @@ class Python3Parser(PythonParser): for i in range(cfl - 1, first, -1): if tokens[i] != "POP_BLOCK": break - if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"): + if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "BREAK_LOOP"): if not tokens[i].kind.startswith("COME_FROM"): return True # Check that the SETUP_LOOP jumps to the offset after the # COME_FROM_LOOP + + # Python 3.0 has additional: + # JUMP_FORWARD here + # COME_FROM + # POP_TOP + # COME_FROM + # here: + # (target of SETUP_LOOP) + # We won't check this. + if self.version == 3.0: + return False + if 0 <= last < len(tokens) and tokens[last] in ( "COME_FROM_LOOP", "JUMP_BACK", diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 432033d6..13160554 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -11,13 +11,21 @@ class Python30Parser(Python31Parser): def p_30(self, args): """ + pt_bp ::= POP_TOP POP_BLOCK + assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP + assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 + come_froms + call_stmt ::= expr _come_froms POP_TOP + return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA # FIXME: combine with parse3.2 - whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK - COME_FROM_LOOP + whileTruestmt ::= SETUP_LOOP l_stmts_opt + JUMP_BACK COME_FROM_LOOP + whileTruestmt ::= SETUP_LOOP l_stmts_opt + CONTINUE COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP returns COME_FROM_LOOP @@ -42,10 +50,22 @@ class Python30Parser(Python31Parser): else_suitel ::= l_stmts COME_FROM_LOOP JUMP_BACK - ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel + jump_absolute_else ::= COME_FROM JUMP_ABSOLUTE COME_FROM POP_TOP + + jump_cf_pop ::= _come_froms _jump _come_froms POP_TOP + + ifelsestmt ::= testexpr c_stmts_opt jump_cf_pop else_suite COME_FROM + ifelsestmtl ::= testexpr c_stmts_opt jump_cf_pop else_suitel + ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec + ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec + + iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM iflaststmtl ::= testexpr c_stmts_opt jb_pop_top + iflaststmtl ::= testexpr c_stmts_opt come_froms JUMP_BACK COME_FROM POP_TOP + iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP + withasstmt ::= expr setupwithas store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_FINALLY LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY @@ -53,8 +73,8 @@ class Python30Parser(Python31Parser): setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST # Need to keep LOAD_FAST as index 1 - set_comp_func_header ::= BUILD_SET_0 DUP_TOP STORE_FAST - set_comp_func ::= set_comp_func_header + set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST + set_comp_func ::= set_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST @@ -62,8 +82,10 @@ class Python30Parser(Python31Parser): list_comp ::= list_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK + list_comp ::= list_comp_header + LOAD_FAST FOR_ITER store comp_iter + JUMP_BACK _come_froms POP_TOP JUMP_BACK - set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST set_comp ::= set_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK @@ -72,6 +94,24 @@ class Python30Parser(Python31Parser): dict_comp ::= dict_comp_header LOAD_FAST FOR_ITER store dict_comp_iter JUMP_BACK + dict_comp ::= dict_comp_header + LOAD_FAST FOR_ITER store dict_comp_iter + JUMP_BACK _come_froms POP_TOP JUMP_BACK + + stmt ::= try_except30 + try_except30 ::= SETUP_EXCEPT suite_stmts_opt + _come_froms pt_bp + except_handler opt_come_from_except + + # From Python 2.6 + + + list_iter ::= list_if JUMP_BACK + list_iter ::= list_if JUMP_BACK _come_froms POP_TOP + lc_body ::= LOAD_NAME expr LIST_APPEND + lc_body ::= LOAD_FAST expr LIST_APPEND + list_if ::= expr jmp_false_then list_iter + ############# dict_comp_iter ::= expr expr ROT_TWO expr STORE_SUBSCR @@ -87,19 +127,52 @@ class Python30Parser(Python31Parser): except_suite ::= c_stmts POP_EXCEPT jump_except POP_TOP except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY _jump COME_FROM POP_TOP - jump_except ::= JUMP_FORWARD COME_FROM POP_TOP - jump_except ::= JUMP_ABSOLUTE COME_FROM POP_TOP + + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY + + _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP + _ifstmts_jump ::= c_stmts_opt come_froms POP_TOP JUMP_FORWARD _come_froms + + jump_except ::= _jump COME_FROM POP_TOP + or ::= expr jmp_false expr jmp_true expr or ::= expr jmp_true expr + import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist _come_froms POP_TOP + ################################################################################ # In many ways 3.0 is like 2.6. One similarity is there is no JUMP_IF_TRUE and # JUMP_IF_FALSE # The below rules in fact are the same or similar. - jmp_true ::= JUMP_IF_TRUE POP_TOP - jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP + jmp_true ::= JUMP_IF_TRUE POP_TOP + jmp_true_then ::= JUMP_IF_TRUE _come_froms POP_TOP + jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP + jmp_false_then ::= JUMP_IF_FALSE POP_TOP + # We don't have hacky THEN detection, so we do it + # in the grammar below which is also somewhat hacky. + + stmt ::= ifstmt30 + stmt ::= ifnotstmt30 + ifstmt30 ::= testfalse_then _ifstmts_jump30 + ifnotstmt30 ::= testtrue_then _ifstmts_jump30 + + testfalse_then ::= expr jmp_false_then + testtrue_then ::= expr jmp_true_then + call_stmt ::= expr COME_FROM + _ifstmts_jump30 ::= c_stmts POP_TOP + + gen_comp_body ::= expr YIELD_VALUE COME_FROM POP_TOP + + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts + COME_FROM POP_TOP END_FINALLY + + or ::= expr jmp_true_then expr come_from_opt + ret_or ::= expr jmp_true_then expr come_from_opt + ret_and ::= expr jump_false expr come_from_opt + + ################################################################################ for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts @@ -107,12 +180,17 @@ class Python30Parser(Python31Parser): except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts POP_TOP END_FINALLY - return_if_stmt ::= ret_expr RETURN_END_IF COME_FROM POP_TOP - and ::= expr jmp_false expr come_from_opt + return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP + return_if_stmt ::= ret_expr RETURN_VALUE come_froms POP_TOP + + and ::= expr jmp_false_then expr come_from_opt + whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt - JUMP_BACK COME_FROM POP_TOP POP_BLOCK COME_FROM_LOOP + JUMP_BACK _come_froms POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP + whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt + come_froms POP_TOP POP_BLOCK COME_FROM_LOOP # compare_chained is like x <= y <= z @@ -123,34 +201,149 @@ class Python30Parser(Python31Parser): compare_chained2 ::= expr COMPARE_OP RETURN_END_IF """ - def customize_grammar_rules(self, tokens, customize): - super(Python30Parser, self).customize_grammar_rules(tokens, customize) + + def remove_rules_30(self): self.remove_rules(""" iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_froms + jump_forward_else ::= JUMP_FORWARD ELSE jump_absolute_else ::= JUMP_ABSOLUTE ELSE whilestmt ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM_LOOP + assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 + return_if_lambda ::= RETURN_END_IF_LAMBDA except_suite ::= c_stmts POP_EXCEPT jump_except whileelsestmt ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP - # No JUMP_IF_FALSE_OR_POP + # No JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP, + # POP_JUMP_IF_FALSE, or POP_JUMP_IF_TRUE + + jmp_false ::= POP_JUMP_IF_FALSE + jmp_true ::= JUMP_IF_TRUE_OR_POP POP_TOP + jmp_true ::= POP_JUMP_IF_TRUE + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained1 COME_FROM compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained2 COME_FROM + ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM + ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM + ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF + COME_FROM ret_expr_or_cond + ret_expr_or_cond ::= ret_cond + or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM + and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM + and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM """) + def customize_grammar_rules(self, tokens, customize): + super(Python30Parser, self).customize_grammar_rules(tokens, customize) + self.remove_rules_30() + + self.check_reduce["iflaststmtl"] = "AST" + self.check_reduce['ifstmt'] = "AST" + self.check_reduce["ifelsestmtc"] = "AST" + self.check_reduce["ifelsestmt"] = "AST" + # self.check_reduce["and"] = "stmt" return + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + invalid = super(Python30Parser, + self).reduce_is_invalid(rule, ast, + tokens, first, last) + if invalid: + return invalid + lhs = rule[0] + if ( + lhs in ("iflaststmtl", "ifstmt", + "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr" + ): + testexpr = ast[0] + if testexpr[0] == "testfalse": + testfalse = testexpr[0] + if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else": + jump_absolute_else = ast[2] + come_from = jump_absolute_else[2] + return come_from == "COME_FROM" and come_from.attr < tokens[first].offset + pass + elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop": + jump_cf_pop = ast[2] + come_froms = jump_cf_pop[0] + for come_from in come_froms: + if come_from.attr < tokens[first].offset: + return True + come_froms = jump_cf_pop[2] + if come_froms == "COME_FROM": + if come_froms.attr < tokens[first].offset: + return True + pass + elif come_froms == "_come_froms": + for come_from in come_froms: + if come_from.attr < tokens[first].offset: + return True + + return False + elif testfalse[1] == "jmp_false": + jmp_false = testfalse[1] + if last == len(tokens): + last -= 1 + while (isinstance(tokens[first].offset, str) and first < last): + first += 1 + if first == last: + return True + while (first < last and isinstance(tokens[last].offset, str)): + last -= 1 + if rule[0] == "iflaststmtl": + return not (jmp_false[0].attr <= tokens[last].offset) + else: + jmp_false_target = jmp_false[0].attr + if tokens[first].offset > jmp_false_target: + return True + return ( + (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD") + pass + pass + pass + # elif lhs == "and": + # return tokens[last+1] == "JUMP_FORWARD" + pass class Python30ParserSingle(Python30Parser, PythonParserSingle): pass + +if __name__ == '__main__': + # Check grammar + p = Python30Parser() + p.remove_rules_30() + p.check_grammar() + from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.0: + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() + from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) + opcode_set = set(s.opc.opname).union(set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split())) + ## FIXME: try this + remain_tokens = set(tokens) - opcode_set + import re + remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) + remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + remain_tokens = set(remain_tokens) - opcode_set + print(remain_tokens) + import sys + if len(sys.argv) > 1: + from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): + print(rule2str(rule[0])) diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index 29bb1598..8eff26dd 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -72,7 +72,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.4: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 32488702..3d169499 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -262,7 +262,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.5: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 90293341..152fbb84 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -440,7 +440,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.6: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index 303b678f..9d45bb76 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -140,31 +140,6 @@ class Python37Parser(Python36Parser): """ def customize_grammar_rules(self, tokens, customize): - self.remove_rules(""" - async_forelse_stmt ::= SETUP_LOOP expr - GET_AITER - LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST - YIELD_FROM - store - POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP - LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE - POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK - JUMP_ABSOLUTE END_FINALLY COME_FROM - for_block POP_BLOCK - else_suite COME_FROM_LOOP - stmt ::= async_for_stmt36 - async_for_stmt36 ::= SETUP_LOOP expr - GET_AITER - LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST - YIELD_FROM - store - POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP - LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE - END_FINALLY continues COME_FROM - POP_TOP POP_TOP POP_TOP POP_EXCEPT - POP_TOP POP_BLOCK - COME_FROM_LOOP - """) super(Python37Parser, self).customize_grammar_rules(tokens, customize) class Python37ParserSingle(Python37Parser, PythonParserSingle): @@ -172,22 +147,33 @@ class Python37ParserSingle(Python37Parser, PythonParserSingle): if __name__ == '__main__': # Check grammar + # FIXME: DRY this with other parseXX.py routines p = Python37Parser() p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.7: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) - opcode_set = set(s.opc.opname).union(set( - """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + opcode_set = set(s.opc.opname).union( + set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LAMBDA_MARKER RETURN_LAST - """.split())) + """.split() + ) + ) remain_tokens = set(tokens) - opcode_set import re - remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + + remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) - # print(sorted(p.rule2name.items())) + import sys + if len(sys.argv) > 1: + from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): + print(rule2str(rule[0])) diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index b137385e..5cea1baf 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -159,7 +159,7 @@ class Python38Parser(Python37Parser): super(Python38Parser, self).__init__(debug_parser) self.customized = {} - def customize_grammar_rules(self, tokens, customize): + def remove_rules_38(self): self.remove_rules(""" stmt ::= async_for_stmt37 stmt ::= for @@ -226,7 +226,10 @@ class Python38Parser(Python37Parser): """) + + def customize_grammar_rules(self, tokens, customize): super(Python37Parser, self).customize_grammar_rules(tokens, customize) + self.remove_rules_38() self.check_reduce['ifstmt'] = 'tokens' self.check_reduce['whileTruestmt38'] = 'tokens' @@ -234,6 +237,7 @@ class Python38Parser(Python37Parser): invalid = super(Python38Parser, self).reduce_is_invalid(rule, ast, tokens, first, last) + self.remove_rules_38() if invalid: return invalid if rule[0] == 'ifstmt': @@ -264,24 +268,34 @@ class Python38Parser(Python37Parser): class Python38ParserSingle(Python38Parser, PythonParserSingle): pass -if __name__ == '__main__': +if __name__ == "__main__": # Check grammar + # FIXME: DRY this with other parseXX.py routines p = Python38Parser() + p.remove_rules_38() p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.8: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) - opcode_set = set(s.opc.opname).union(set( - """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM - LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME - LAMBDA_MARKER RETURN_LAST + opcode_set = set(s.opc.opname).union( + set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST """.split())) remain_tokens = set(tokens) - opcode_set import re - remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + + remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) - # print(sorted(p.rule2name.items())) + import sys + if len(sys.argv) > 1: + from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): + print(rule2str(rule[0])) diff --git a/uncompyle6/parsers/treenode.py b/uncompyle6/parsers/treenode.py index f4a57559..aec8d06e 100644 --- a/uncompyle6/parsers/treenode.py +++ b/uncompyle6/parsers/treenode.py @@ -54,3 +54,11 @@ class SyntaxTree(spark_AST): rv += "\n" + child i += 1 return rv + + def first_child(self): + if len(self) > 0: + child = self[0] + if not isinstance(child, SyntaxTree): + return child + return self[0].first_child() + return self diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 6c171393..35ac9371 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -265,21 +265,29 @@ class Scanner3(Scanner): # RAISE_VARARGS then we have a "raise" statement # else we have an "assert" statement. if self.version == 3.0: - # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here + # Like 2.6, 3.0 doesn't have POP_JUMP_IF... so we have + # to go through more machinations assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n + if assert_can_follow: + prev_inst = self.insts[i - 1] + assert_can_follow = ( + prev_inst.opname in ("JUMP_IF_TRUE", "JUMP_IF_FALSE") + and i + 1 < n ) + jump_if_inst = prev_inst else: assert_can_follow = ( inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE") and i + 1 < n ) + jump_if_inst = inst if assert_can_follow: next_inst = self.insts[i + 1] if ( next_inst.opname == "LOAD_GLOBAL" and next_inst.argval == "AssertionError" - and inst.argval + and jump_if_inst.argval ): - raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] + raise_idx = self.offset2inst_index[self.prev_op[jump_if_inst.argval]] raise_inst = self.insts[raise_idx] if raise_inst.opname.startswith("RAISE_VARARGS"): self.load_asserts.add(next_inst.offset) @@ -482,6 +490,12 @@ class Scanner3(Scanner): and self.insts[i + 1].opname == "JUMP_FORWARD" ) + if (self.version == 3.0 and self.insts[i + 1].opname == "JUMP_FORWARD" + and not is_continue): + target_prev = self.offset2inst_index[self.prev_op[target]] + is_continue = ( + self.insts[target_prev].opname == "SETUP_LOOP") + if is_continue or ( inst.offset in self.stmts and ( diff --git a/uncompyle6/semantics/customize26_27.py b/uncompyle6/semantics/customize26_27.py index 324b50b8..a57920a8 100644 --- a/uncompyle6/semantics/customize26_27.py +++ b/uncompyle6/semantics/customize26_27.py @@ -40,6 +40,7 @@ def customize_for_version26_27(self, version): 'testtrue_then': ( 'not %p', (0, 22) ), }) + # FIXME: this should be a transformation def n_call(node): mapping = self._get_mapping(node) key = node diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index 58084b50..0734145a 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -181,6 +181,19 @@ def customize_for_version3(self, version): # the iteration variable. These rules we can ignore # since we pick up the iteration variable some other way and # we definitely don't include in the source _[dd]. + TABLE_DIRECT.update({ + "ifstmt30": ( "%|if %c:\n%+%c%-", + (0, "testfalse_then"), + (1, "_ifstmts_jump30") ), + "ifnotstmt30": ( "%|if not %c:\n%+%c%-", + (0, "testtrue_then"), + (1, "_ifstmts_jump30") ), + "try_except30": ( "%|try:\n%+%c%-%c\n\n", + (1, "suite_stmts_opt"), + (4, "except_handler") ), + + }) + def n_comp_iter(node): if node[0] == "expr": n = node[0][0] @@ -197,11 +210,14 @@ def customize_for_version3(self, version): # FIXME: perhaps this can be folded into the 3.4+ case? def n_yield_from(node): assert node[0] == "expr" - assert node[0][0] == "get_iter" - # Skip over yield_from.expr.get_iter which adds an - # extra iter(). Maybe we can do in tranformation phase instead? - template = ("yield from %c", (0, "expr")) - self.template_engine(template, node[0][0]) + if node[0][0] == "get_iter": + # Skip over yield_from.expr.get_iter which adds an + # extra iter(). Maybe we can do in tranformation phase instead? + template = ("yield from %c", (0, "expr")) + self.template_engine(template, node[0][0]) + else: + template = ("yield from %c", (0, "attribute")) + self.template_engine(template, node[0][0][0]) self.prune() self.n_yield_from = n_yield_from diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 5048dc70..0d201064 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -792,7 +792,7 @@ class FragmentsWalker(pysource.SourceWalker, object): self.name = code_name # Issue created with later Python code generation is that there - # is a lamda set up with a dummy argument name that is then called + # is a lambda set up with a dummy argument name that is then called # So we can't just translate that as is but need to replace the # dummy name. Below we are picking out the variable name as seen # in the code. And trying to generate code for the other parts diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index 7156db40..f80fe23d 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -115,11 +115,14 @@ class TreeTransform(GenericASTTraversal, object): call = expr[0] LOAD_ASSERT = call[0] - expr = call[1][0] - node = SyntaxTree( - kind, - [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1] - ) + if isinstance(call[1], SyntaxTree): + expr = call[1][0] + node = SyntaxTree( + kind, + [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1] + ) + pass + pass else: # ifstmt # 0. testexpr (2)