From 078cca335accecf1e9d8c37d145b56dfaaea4b39 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 6 Jan 2020 14:57:11 -0500 Subject: [PATCH] Seomtiems we need to add "yield"... in order to get the generator bit flag set, such as in 3.x where the yield is optimized away. --- test/stdlib/runtests.sh | 2 -- uncompyle6/semantics/make_function2.py | 22 +++++++++++++++++++--- uncompyle6/semantics/make_function3.py | 20 ++++++++++++++++++-- uncompyle6/semantics/make_function36.py | 21 ++++++++++++++++++--- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 7e5395be..dcfc9b78 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -271,7 +271,6 @@ case $PYVERSION in [test_compare.py]=1 # test assert fail - investigate [test_compile.py]=1 [test_configparser.py]=1 - [test_contains.py]=1 # Code "while False: yield None" is optimized away in compilation [test_contextlib_async.py]=1 # Investigate [test_context.py]=1 [test_coroutines.py]=1 # Parse error @@ -293,7 +292,6 @@ case $PYVERSION in ;; 3.8) SKIP_TESTS=( - [test_contains.py]=1 # Code "while False: yield None" is optimized away in compilation [test_collections.py]=1 # Investigate [test_decorators.py]=1 # Control flow wrt "if elif" [test_exceptions.py]=1 # parse error diff --git a/uncompyle6/semantics/make_function2.py b/uncompyle6/semantics/make_function2.py index 10fd0e87..0e53ff45 100644 --- a/uncompyle6/semantics/make_function2.py +++ b/uncompyle6/semantics/make_function2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2019 by Rocky Bernstein +# Copyright (c) 2015-2020 by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # This program is free software: you can redistribute it and/or modify @@ -18,6 +18,7 @@ All the crazy things we have to do to handle Python functions in Python before 3 The saga of changes continues in 3.0 and above and in other files. """ from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg +from xdis.util import CO_GENERATOR from uncompyle6.scanner import Code from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6 import PYTHON3 @@ -182,7 +183,6 @@ def make_function2(self, node, is_lambda, nested=1, code_node=None): # docstring exists, dump it print_docstring(self, indent, code.co_consts[0]) - code._tokens = None # save memory if not is_lambda: assert ast == "stmts" @@ -203,5 +203,21 @@ def make_function2(self, node, is_lambda, nested=1, code_node=None): self.gen_source( ast, code.co_name, code._customize, is_lambda=is_lambda, returnNone=rn ) - code._tokens = None + + # In obscure cases, a function may be a generator but the "yield" + # was optimized away. Here, we need to put in unreachable code to + # add in "yield" just so that the compiler will mark + # the GENERATOR bit of the function. See for example + # Python 3.x's test_generator.py test program. + if code.co_flags & CO_GENERATOR: + need_bogus_yield = True + for token in scanner_code._tokens: + if token == "YIELD_VALUE": + need_bogus_yield = False + break + pass + if need_bogus_yield: + self.template_engine(("%|if False:\n%+%|yield None%-",), node) + + code._tokens = None # save memory code._customize = None # save memory diff --git a/uncompyle6/semantics/make_function3.py b/uncompyle6/semantics/make_function3.py index 91eae46c..600681c1 100644 --- a/uncompyle6/semantics/make_function3.py +++ b/uncompyle6/semantics/make_function3.py @@ -17,6 +17,7 @@ All the crazy things we have to do to handle Python functions in 3.0-3.5 or so. The saga of changes before and after is in other files. """ from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg +from xdis.util import CO_GENERATOR from uncompyle6.scanner import Code from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6 import PYTHON3 @@ -645,7 +646,6 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None): # docstring exists, dump it print_docstring(self, self.indent, code.co_consts[0]) - scanner_code._tokens = None # save memory assert ast == "stmts" all_globals = find_all_globals(ast, set()) @@ -665,5 +665,21 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None): self.gen_source( ast, code.co_name, scanner_code._customize, is_lambda=is_lambda, returnNone=rn ) - scanner_code._tokens = None + + # In obscure cases, a function may be a generator but the "yield" + # was optimized away. Here, we need to put in unreachable code to + # add in "yield" just so that the compiler will mark + # the GENERATOR bit of the function. See for example + # Python 3.x's test_generator.py test program. + if code.co_flags & CO_GENERATOR: + need_bogus_yield = True + for token in scanner_code._tokens: + if token in ("YIELD_VALUE", "YIELD_FROM"): + need_bogus_yield = False + break + pass + if need_bogus_yield: + self.template_engine(("%|if False:\n%+%|yield None%-",), node) + + scanner_code._tokens = None # save memory scanner_code._customize = None # save memory diff --git a/uncompyle6/semantics/make_function36.py b/uncompyle6/semantics/make_function36.py index 142d7601..b0abfe43 100644 --- a/uncompyle6/semantics/make_function36.py +++ b/uncompyle6/semantics/make_function36.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 by Rocky Bernstein +# Copyright (c) 2019-2020 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,6 +17,7 @@ All the crazy things we have to do to handle Python functions in 3.6 and above. The saga of changes before 3.6 is in other files. """ from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg +from xdis.util import CO_GENERATOR from uncompyle6.scanner import Code from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.semantics.parser_error import ParserError @@ -371,7 +372,6 @@ def make_function36(self, node, is_lambda, nested=1, code_node=None): # docstring exists, dump it self.println(self.traverse(node[-2])) - scanner_code._tokens = None # save memory assert ast == "stmts" all_globals = find_all_globals(ast, set()) @@ -392,5 +392,20 @@ def make_function36(self, node, is_lambda, nested=1, code_node=None): ast, code.co_name, scanner_code._customize, is_lambda=is_lambda, returnNone=rn ) - scanner_code._tokens = None + # In obscure cases, a function may be a generator but the "yield" + # was optimized away. Here, we need to put in unreachable code to + # add in "yield" just so that the compiler will mark + # the GENERATOR bit of the function. See for example + # Python 3.x's test_generator.py test program. + if code.co_flags & CO_GENERATOR: + need_bogus_yield = True + for token in scanner_code._tokens: + if token == "YIELD_VALUE": + need_bogus_yield = False + break + pass + if need_bogus_yield: + self.template_engine(("%|if False:\n%+%|yield None%-",), node) + + scanner_code._tokens = None # save memory scanner_code._customize = None # save memory