Seomtiems we need to add "yield"...

in order to get the generator bit flag set, such as in 3.x where the
yield is optimized away.

test/stdlib/runtests.sh

@@ -271,7 +271,6 @@ case $PYVERSION in
 	    [test_compare.py]=1 # test assert fail - investigate
 	    [test_compile.py]=1
 	    [test_configparser.py]=1
-	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation
 	    [test_contextlib_async.py]=1 # Investigate
 	    [test_context.py]=1
 	    [test_coroutines.py]=1 # Parse error
@@ -293,7 +292,6 @@ case $PYVERSION in
 	;;
     3.8)
 	SKIP_TESTS=(
-	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation
 	    [test_collections.py]=1  # Investigate
 	    [test_decorators.py]=1  # Control flow wrt "if elif"
 	    [test_exceptions.py]=1   # parse error

uncompyle6/semantics/make_function2.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2019 by Rocky Bernstein
+#  Copyright (c) 2015-2020 by Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
 #  This program is free software: you can redistribute it and/or modify
@@ -18,6 +18,7 @@ All the crazy things we have to do to handle Python functions in Python before 3
 The saga of changes continues in 3.0 and above and in other files.
 """
 from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg
+from xdis.util import CO_GENERATOR
 from uncompyle6.scanner import Code
 from uncompyle6.parsers.treenode import SyntaxTree
 from uncompyle6 import PYTHON3
@@ -182,7 +183,6 @@ def make_function2(self, node, is_lambda, nested=1, code_node=None):
         # docstring exists, dump it
         print_docstring(self, indent, code.co_consts[0])
 
-    code._tokens = None  # save memory
     if not is_lambda:
         assert ast == "stmts"
 
@@ -203,5 +203,21 @@ def make_function2(self, node, is_lambda, nested=1, code_node=None):
     self.gen_source(
         ast, code.co_name, code._customize, is_lambda=is_lambda, returnNone=rn
     )
-    code._tokens = None
+
+    # In obscure cases, a function may be a generator but the "yield"
+    # was optimized away. Here, we need to put in unreachable code to
+    # add in "yield" just so that the compiler will mark
+    # the GENERATOR bit of the function. See for example
+    # Python 3.x's test_generator.py test program.
+    if code.co_flags & CO_GENERATOR:
+        need_bogus_yield = True
+        for token in scanner_code._tokens:
+            if token == "YIELD_VALUE":
+                need_bogus_yield = False
+                break
+            pass
+        if need_bogus_yield:
+            self.template_engine(("%|if False:\n%+%|yield None%-",), node)
+
+    code._tokens = None # save memory
     code._customize = None  # save memory

uncompyle6/semantics/make_function3.py

@@ -17,6 +17,7 @@ All the crazy things we have to do to handle Python functions in 3.0-3.5 or so.
 The saga of changes before and after is in other files.
 """
 from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg
+from xdis.util import CO_GENERATOR
 from uncompyle6.scanner import Code
 from uncompyle6.parsers.treenode import SyntaxTree
 from uncompyle6 import PYTHON3
@@ -645,7 +646,6 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
         # docstring exists, dump it
         print_docstring(self, self.indent, code.co_consts[0])
 
-    scanner_code._tokens = None  # save memory
     assert ast == "stmts"
 
     all_globals = find_all_globals(ast, set())
@@ -665,5 +665,21 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     self.gen_source(
         ast, code.co_name, scanner_code._customize, is_lambda=is_lambda, returnNone=rn
     )
-    scanner_code._tokens = None
+
+    # In obscure cases, a function may be a generator but the "yield"
+    # was optimized away. Here, we need to put in unreachable code to
+    # add in "yield" just so that the compiler will mark
+    # the GENERATOR bit of the function. See for example
+    # Python 3.x's test_generator.py test program.
+    if code.co_flags & CO_GENERATOR:
+        need_bogus_yield = True
+        for token in scanner_code._tokens:
+            if token in ("YIELD_VALUE", "YIELD_FROM"):
+                need_bogus_yield = False
+                break
+            pass
+        if need_bogus_yield:
+            self.template_engine(("%|if False:\n%+%|yield None%-",), node)
+
+    scanner_code._tokens = None # save memory
     scanner_code._customize = None  # save memory

uncompyle6/semantics/make_function36.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2019 by Rocky Bernstein
+#  Copyright (c) 2019-2020 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,6 +17,7 @@ All the crazy things we have to do to handle Python functions in 3.6 and above.
 The saga of changes before 3.6 is in other files.
 """
 from xdis.code import iscode, code_has_star_arg, code_has_star_star_arg
+from xdis.util import CO_GENERATOR
 from uncompyle6.scanner import Code
 from uncompyle6.parsers.treenode import SyntaxTree
 from uncompyle6.semantics.parser_error import ParserError
@@ -371,7 +372,6 @@ def make_function36(self, node, is_lambda, nested=1, code_node=None):
         # docstring exists, dump it
         self.println(self.traverse(node[-2]))
 
-    scanner_code._tokens = None  # save memory
     assert ast == "stmts"
 
     all_globals = find_all_globals(ast, set())
@@ -392,5 +392,20 @@ def make_function36(self, node, is_lambda, nested=1, code_node=None):
         ast, code.co_name, scanner_code._customize, is_lambda=is_lambda, returnNone=rn
     )
 
-    scanner_code._tokens = None
+    # In obscure cases, a function may be a generator but the "yield"
+    # was optimized away. Here, we need to put in unreachable code to
+    # add in "yield" just so that the compiler will mark
+    # the GENERATOR bit of the function. See for example
+    # Python 3.x's test_generator.py test program.
+    if code.co_flags & CO_GENERATOR:
+        need_bogus_yield = True
+        for token in scanner_code._tokens:
+            if token == "YIELD_VALUE":
+                need_bogus_yield = False
+                break
+            pass
+        if need_bogus_yield:
+            self.template_engine(("%|if False:\n%+%|yield None%-",), node)
+
+    scanner_code._tokens = None # save memory
     scanner_code._customize = None  # save memory