From 5c662b334e72ad1a6e35a150a0ac54ba1f2273e5 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 19 Mar 2018 07:55:09 -0400 Subject: [PATCH] Handle 3.x "nonlocal" statement --- test/bytecode_3.3_run/05_nonlocal.pyc | Bin 0 -> 560 bytes test/simple_source/bug33/05_nonlocal.py | 12 +++++++ uncompyle6/semantics/helper.py | 18 +++++++--- uncompyle6/semantics/make_function.py | 43 ++++++++++++++++++------ uncompyle6/semantics/pysource.py | 16 +++++++-- 5 files changed, 71 insertions(+), 18 deletions(-) create mode 100644 test/bytecode_3.3_run/05_nonlocal.pyc create mode 100644 test/simple_source/bug33/05_nonlocal.py diff --git a/test/bytecode_3.3_run/05_nonlocal.pyc b/test/bytecode_3.3_run/05_nonlocal.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22dc2026853695c2b1325b44710eefe0328712fe GIT binary patch literal 560 zcma)2yH3ME5S(*DfDkC4LKH|;t{kL@E{dR_O_MH-73?Eeaqgl!Ux5S-AYLDV_yhim z?_l>V%!w+NceI|HxgGCEZ@2sY`8E9pkWn>HMBO=+JtoNM6?g)sfCL_iK0+n%Adp9x z#4XY^Zb2VRaMf;sH{b=h_VT=iXW$;V^EXZ6HVMbY24Qmt_22{tRJKd->Juz@81uC} zrVcgmP`5*n&9WPrR(c_oKXE5S+G4pXWoorEne1P=`C!mLJxi5VrOsw$vbuF!Br|ee z*h-oXTX+m2cSxvom69nLdbJ%TzrtX4=nv?Bk=9MK`8Udi$A>;N) jcVVqGRiV|TG1}B&*zdb_6Y&2GmpXT)9Qg>mkKXnVA1QD8 literal 0 HcmV?d00001 diff --git a/test/simple_source/bug33/05_nonlocal.py b/test/simple_source/bug33/05_nonlocal.py new file mode 100644 index 00000000..8506ff07 --- /dev/null +++ b/test/simple_source/bug33/05_nonlocal.py @@ -0,0 +1,12 @@ +# From Python 3.6 functools.py +# Bug was in detecting "nonlocal" access +def not_bug(): + cache_token = 5 + + def register(): + nonlocal cache_token + return cache_token == 5 + + return register() + +assert not_bug() diff --git a/uncompyle6/semantics/helper.py b/uncompyle6/semantics/helper.py index 970595f3..248c0016 100644 --- a/uncompyle6/semantics/helper.py +++ b/uncompyle6/semantics/helper.py @@ -13,6 +13,9 @@ else: read_write_global_ops = frozenset(('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL')) read_global_ops = frozenset(('STORE_GLOBAL', 'DELETE_GLOBAL')) +# NOTE: we also need to check that he variable name is a free variable, not a cell variable. +nonglobal_ops = frozenset(('LOAD_DEREF', 'STORE_DEREF', 'DELETE_DEREF')) + # FIXME: this and find_globals could be paramaterized with one of the # above global ops def find_all_globals(node, globs): @@ -24,15 +27,22 @@ def find_all_globals(node, globs): globs.add(n.pattr) return globs -def find_globals(node, globs): +def find_globals_and_nonlocals(node, globs, nonlocals, code, version): """search a node of parse tree to find variable names that need a - 'global' added.""" + either 'global' or 'nonlocal' statements added.""" for n in node: if isinstance(n, AST): - globs = find_globals(n, globs) + globs, nonlocals = find_globals_and_nonlocals(n, globs, nonlocals, + code, version) elif n.kind in read_global_ops: globs.add(n.pattr) - return globs + elif (version >= 3.0 + and n.kind in nonglobal_ops + and n.pattr in code.co_freevars + and n.pattr != code.co_name + and code.co_name != ''): + nonlocals.add(n.pattr) + return globs, nonlocals # def find_globals(node, globs, global_ops=mkfunc_globals): # """Find globals in this statement.""" diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 41f16cdd..5587f991 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -23,7 +23,7 @@ from uncompyle6 import PYTHON3 from uncompyle6.semantics.parser_error import ParserError from uncompyle6.parser import ParserError as ParserError2 from uncompyle6.semantics.helper import ( - print_docstring, find_all_globals, find_globals, find_none + print_docstring, find_all_globals, find_globals_and_nonlocals, find_none ) if PYTHON3: @@ -269,8 +269,12 @@ def make_function3_annotate(self, node, is_lambda, nested=1, assert ast == 'stmts' all_globals = find_all_globals(ast, set()) - for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())): + globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(), + code, self.version) + for g in sorted((all_globals & self.mod_globs) | globals): self.println(self.indent, 'global ', g) + for nl in sorted(nonlocals): + self.println(self.indent, 'nonlocal ', nl) self.mod_globs -= all_globals has_none = 'None' in code.co_names rn = has_none and not find_none(ast) @@ -422,7 +426,17 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None): assert ast == 'stmts' all_globals = find_all_globals(ast, set()) - for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())): + + globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(), + code, self.version) + + # Python 2 doesn't support the "nonlocal" statement + try: + assert self.version >= 3.0 or not nonlocals + except: + from trepan.api import debug; debug() + + for g in sorted((all_globals & self.mod_globs) | globals): self.println(self.indent, 'global ', g) self.mod_globs -= all_globals has_none = 'None' in code.co_names @@ -536,19 +550,19 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None): code = codeNode.attr assert iscode(code) - code = Code(code, self.scanner, self.currentclass) + scanner_code = Code(code, self.scanner, self.currentclass) # add defaults values to parameter names argc = code.co_argcount - paramnames = list(code.co_varnames[:argc]) + paramnames = list(scanner_code.co_varnames[:argc]) # defaults are for last n parameters, thus reverse if not 3.0 <= self.version <= 3.1 or self.version >= 3.6: paramnames.reverse(); defparams.reverse() try: - ast = self.build_ast(code._tokens, - code._customize, + ast = self.build_ast(scanner_code._tokens, + scanner_code._customize, is_lambda = is_lambda, noneInNames = ('None' in code.co_names)) except (ParserError, ParserError2) as p: @@ -703,15 +717,22 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None): # docstring exists, dump it print_docstring(self, self.indent, code.co_consts[0]) - code._tokens = None # save memory + scanner_code._tokens = None # save memory assert ast == 'stmts' all_globals = find_all_globals(ast, set()) - for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())): + globals, nonlocals = find_globals_and_nonlocals(ast, set(), + set(), code, self.version) + + for g in sorted((all_globals & self.mod_globs) | globals): self.println(self.indent, 'global ', g) + + for nl in sorted(nonlocals): + self.println(self.indent, 'nonlocal ', nl) + self.mod_globs -= all_globals has_none = 'None' in code.co_names rn = has_none and not find_none(ast) - self.gen_source(ast, code.co_name, code._customize, is_lambda=is_lambda, + self.gen_source(ast, code.co_name, scanner_code._customize, is_lambda=is_lambda, returnNone=rn) - code._tokens = None; code._customize = None # save memory + scanner_code._tokens = None; scanner_code._customize = None # save memory diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index caf3551c..917e48a4 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -141,7 +141,7 @@ from uncompyle6.semantics.parser_error import ParserError from uncompyle6.semantics.check_ast import checker from uncompyle6.semantics.customize import customize_for_version from uncompyle6.semantics.helper import ( - print_docstring, find_globals, flatten_list) + print_docstring, find_globals_and_nonlocals, flatten_list) from uncompyle6.scanners.tok import Token from uncompyle6.semantics.consts import ( @@ -581,6 +581,7 @@ class SourceWalker(GenericASTTraversal, object): self.pending_newlines = 0 self.params = { '_globals': {}, + '_nonlocals': {}, # Python 3 has nonlocal 'f': StringIO(), 'indent': indent, 'is_lambda': is_lambda, @@ -2303,11 +2304,16 @@ class SourceWalker(GenericASTTraversal, object): # else: # print ast[-1][-1] + globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(), + code, self.version) # Add "global" declaration statements at the top # of the function - for g in sorted(find_globals(ast, set())): + for g in sorted(globals): self.println(indent, 'global ', g) + for nl in sorted(nonlocals): + self.println(indent, 'nonlocal ', nl) + old_name = self.name self.gen_source(ast, code.co_name, code._customize) self.name = old_name @@ -2461,7 +2467,11 @@ def code_deparse(co, out=sys.stdout, version=None, debug_opts=DEFAULT_DEBUG_OPTS # save memory del tokens - deparsed.mod_globs = find_globals(deparsed.ast, set()) + deparsed.mod_globs, nonlocals = find_globals_and_nonlocals(deparsed.ast, + set(), set(), + co, version) + + assert not nonlocals # convert leading '__doc__ = "..." into doc string try: