Handle 3.x "nonlocal" statement

test/simple_source/bug33/05_nonlocal.py

@@ -0,0 +1,12 @@
+# From Python 3.6 functools.py
+# Bug was in detecting "nonlocal" access
+def not_bug():
+    cache_token = 5
+
+    def register():
+        nonlocal cache_token
+        return cache_token == 5
+
+    return register()
+
+assert not_bug()

uncompyle6/semantics/helper.py

@@ -13,6 +13,9 @@ else:
 read_write_global_ops = frozenset(('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'))
 read_global_ops       = frozenset(('STORE_GLOBAL', 'DELETE_GLOBAL'))
 
+# NOTE: we also need to check that he variable name is a free variable, not a cell variable.
+nonglobal_ops         = frozenset(('LOAD_DEREF', 'STORE_DEREF',  'DELETE_DEREF'))
+
 # FIXME: this and find_globals could be paramaterized with one of the
 # above global ops
 def find_all_globals(node, globs):
@@ -24,15 +27,22 @@ def find_all_globals(node, globs):
             globs.add(n.pattr)
     return globs
 
-def find_globals(node, globs):
+def find_globals_and_nonlocals(node, globs, nonlocals, code, version):
     """search a node of parse tree to find variable names that need a
-    'global' added."""
+    either 'global' or 'nonlocal' statements added."""
     for n in node:
         if isinstance(n, AST):
-            globs = find_globals(n, globs)
+            globs, nonlocals = find_globals_and_nonlocals(n, globs, nonlocals,
+                                                          code, version)
         elif n.kind in read_global_ops:
             globs.add(n.pattr)
-    return globs
+        elif (version >= 3.0
+              and n.kind in nonglobal_ops
+              and n.pattr in code.co_freevars
+              and n.pattr != code.co_name
+              and code.co_name != '<lambda>'):
+            nonlocals.add(n.pattr)
+    return globs, nonlocals
 
 # def find_globals(node, globs, global_ops=mkfunc_globals):
 #     """Find globals in this statement."""

uncompyle6/semantics/make_function.py

@@ -23,7 +23,7 @@ from uncompyle6 import PYTHON3
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.parser import ParserError as ParserError2
 from uncompyle6.semantics.helper import (
-    print_docstring, find_all_globals, find_globals, find_none
+    print_docstring, find_all_globals, find_globals_and_nonlocals, find_none
     )
 
 if PYTHON3:
@@ -269,8 +269,12 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
     assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
-    for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())):
+    globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),
+                                                    code, self.version)
+    for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
+    for nl in sorted(nonlocals):
+        self.println(self.indent, 'nonlocal ', nl)
     self.mod_globs -= all_globals
     has_none = 'None' in code.co_names
     rn = has_none and not find_none(ast)
@@ -422,7 +426,17 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None):
         assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
-    for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())):
+
+    globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),
+                                                    code, self.version)
+
+    # Python 2 doesn't support the "nonlocal" statement
+    try:
+        assert self.version >= 3.0 or not nonlocals
+    except:
+        from trepan.api import debug; debug()
+
+    for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
     self.mod_globs -= all_globals
     has_none = 'None' in code.co_names
@@ -536,19 +550,19 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         code = codeNode.attr
 
     assert iscode(code)
-    code = Code(code, self.scanner, self.currentclass)
+    scanner_code = Code(code, self.scanner, self.currentclass)
 
     # add defaults values to parameter names
     argc = code.co_argcount
-    paramnames = list(code.co_varnames[:argc])
+    paramnames = list(scanner_code.co_varnames[:argc])
 
     # defaults are for last n parameters, thus reverse
     if not 3.0 <= self.version <= 3.1 or self.version >= 3.6:
         paramnames.reverse(); defparams.reverse()
 
     try:
-        ast = self.build_ast(code._tokens,
+        ast = self.build_ast(scanner_code._tokens,
-                             code._customize,
+                             scanner_code._customize,
                              is_lambda = is_lambda,
                              noneInNames = ('None' in code.co_names))
     except (ParserError, ParserError2) as p:
@@ -703,15 +717,22 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         # docstring exists, dump it
         print_docstring(self, self.indent, code.co_consts[0])
 
-    code._tokens = None # save memory
+    scanner_code._tokens = None # save memory
     assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
-    for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())):
+    globals, nonlocals = find_globals_and_nonlocals(ast, set(),
+                                                    set(), code, self.version)
+
+    for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
+
+    for nl in sorted(nonlocals):
+        self.println(self.indent, 'nonlocal ', nl)
+
     self.mod_globs -= all_globals
     has_none = 'None' in code.co_names
     rn = has_none and not find_none(ast)
-    self.gen_source(ast, code.co_name, code._customize, is_lambda=is_lambda,
+    self.gen_source(ast, code.co_name, scanner_code._customize, is_lambda=is_lambda,
                     returnNone=rn)
-    code._tokens = None; code._customize = None # save memory
+    scanner_code._tokens = None; scanner_code._customize = None # save memory

uncompyle6/semantics/pysource.py

@@ -141,7 +141,7 @@ from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.check_ast import checker
 from uncompyle6.semantics.customize import customize_for_version
 from uncompyle6.semantics.helper import (
-    print_docstring, find_globals, flatten_list)
+    print_docstring, find_globals_and_nonlocals, flatten_list)
 from uncompyle6.scanners.tok import Token
 
 from uncompyle6.semantics.consts import (
@@ -581,6 +581,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.pending_newlines = 0
         self.params = {
             '_globals': {},
+            '_nonlocals': {},   # Python 3 has nonlocal
             'f': StringIO(),
             'indent': indent,
             'is_lambda': is_lambda,
@@ -2303,11 +2304,16 @@ class SourceWalker(GenericASTTraversal, object):
         # else:
         #    print ast[-1][-1]
 
+        globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),
+                                                        code, self.version)
         # Add "global" declaration statements at the top
         # of the function
-        for g in sorted(find_globals(ast, set())):
+        for g in sorted(globals):
             self.println(indent, 'global ', g)
 
+        for nl in sorted(nonlocals):
+            self.println(indent, 'nonlocal ', nl)
+
         old_name = self.name
         self.gen_source(ast, code.co_name, code._customize)
         self.name = old_name
@@ -2461,7 +2467,11 @@ def code_deparse(co, out=sys.stdout, version=None, debug_opts=DEFAULT_DEBUG_OPTS
     # save memory
     del tokens
 
-    deparsed.mod_globs = find_globals(deparsed.ast, set())
+    deparsed.mod_globs, nonlocals = find_globals_and_nonlocals(deparsed.ast,
+                                                               set(), set(),
+                                                               co, version)
+
+    assert not nonlocals
 
     # convert leading '__doc__ = "..." into doc string
     try: