From b128e4fde6708caf847c7c0eb5f494ebd6c9e5f4 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 28 Feb 2018 11:10:54 -0500 Subject: [PATCH] Fix 3.6+ nested list comprehensions Modified "load_closure" rule and changed semantic actions since LOAD_CLOSUREs are stored inside a MAKE_TUPLE. FIXME: Not sure if we have additional "if" clauses correct. Test and correct --- test/bytecode_3.6/06_listcomp_nest.pyc | Bin 0 -> 660 bytes test/simple_source/bug36/06_listcomp_nest.py | 18 +++++ uncompyle6/parsers/parse3.py | 7 +- uncompyle6/semantics/pysource.py | 73 +++++++++++++------ 4 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 test/bytecode_3.6/06_listcomp_nest.pyc create mode 100644 test/simple_source/bug36/06_listcomp_nest.py diff --git a/test/bytecode_3.6/06_listcomp_nest.pyc b/test/bytecode_3.6/06_listcomp_nest.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de374761614c5ccb7a38d7024676352507e87ab7 GIT binary patch literal 660 zcmYjO!A=`75S_7|#H4^w%c2eZ6#fB3dF zddDLvf-VaDage6Pv{nJ%Y=Vw{5V_>o9LJQdLon_bSKQk~IyTJp2tx0^E|-<{vC;9(Zi`@-!gCdm zuCb-nL3vx|ZKGnVB2iFepLcz#4wwg#pSI;2*Tlof9`u_HT<;kGw^55-)Y9$I*Cmb5 X?{*?K|Nm@353AZNOFN3)bOwI`wK{`b literal 0 HcmV?d00001 diff --git a/test/simple_source/bug36/06_listcomp_nest.py b/test/simple_source/bug36/06_listcomp_nest.py new file mode 100644 index 00000000..1bd65360 --- /dev/null +++ b/test/simple_source/bug36/06_listcomp_nest.py @@ -0,0 +1,18 @@ +# From 3.6 _sitebuiltins.py +# Bug was in handling double nested kinds of things like: +# for a in b for c in d + +# This required grammar modification and +# and semantic action changes. LOAD_CLOSUREs are stored +# inside a MAKE_TUPLE. + +# FIXME: test and try additional "if" clauses. +def __init__(self, path, name, files=(), dirs=(), volumes=()): + f = [path.join(dir, filename) + for dir in dirs + for filename in files] + f2 = [path.join(drive, dir, filename) + for dir in dirs + for filename in files + for drive in volumes] + return f, f2 diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 68519a55..2b74a2d2 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -885,8 +885,13 @@ class Python3Parser(PythonParser): "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname)) self.add_make_function_rule(rule_pat, opname, token.attr, customize) if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LISTCOMP'): - rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr " + if self.version < 3.6: + rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname)) + else: + # 3.6+ bundles all of the 'exprs' in the rule above into a tuple. + rule_pat = ("listcomp ::= load_closure LOAD_LISTCOMP %%s%s expr " + "GET_ITER CALL_FUNCTION_1" % (opname,)) self.add_make_function_rule(rule_pat, opname, token.attr, customize) if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LAMBDA'): diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index d7fc20d8..7cb908b8 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1532,10 +1532,9 @@ class SourceWalker(GenericASTTraversal, object): self.prune() def comprehension_walk3(self, node, iter_index, code_index=-5): - """ - List comprehensions the way they are done in Python3. - They are other comprehensions, e.g. set comprehensions - See if we can combine code. + """Non-closure-based comprehensions the way they are done in Python3. + They are other comprehensions, e.g. set comprehensions See if + we can combine code. """ p = self.prec self.prec = 27 @@ -1600,7 +1599,7 @@ class SourceWalker(GenericASTTraversal, object): assert n.kind in ('lc_body', 'comp_body', 'setcomp_func', 'set_comp_body'), ast assert store, "Couldn't find store in list/set comprehension" - # Issue created with later Python code generation is that there + # A problem created with later Python code generation is that there # is a lamda set up with a dummy argument name that is then called # So we can't just translate that as is but need to replace the # dummy name. Below we are picking out the variable name as seen @@ -1641,8 +1640,8 @@ class SourceWalker(GenericASTTraversal, object): self.prec = p def listcomprehension_walk2(self, node): - """List comprehensions the way they are done in Python 2 (and - some Python 3?). + """List comprehensions the way they are done in Python 2 and + sometimes in Python 3. They're more other comprehensions, e.g. set comprehensions See if we can combine code. """ @@ -1652,32 +1651,46 @@ class SourceWalker(GenericASTTraversal, object): code = Code(node[1].attr, self.scanner, self.currentclass) ast = self.build_ast(code._tokens, code._customize) self.customize(code._customize) - if node == 'set_comp': - ast = ast[0][0][0] - else: - ast = ast[0][0][0][0][0] - if ast == 'expr': + # skip over: sstmt, stmt, return, ret_expr + # and other singleton derivations + while (len(ast) == 1 + or (ast in ('sstmt', 'return') + and ast[-1] in ('RETURN_LAST', 'RETURN_VALUE'))): ast = ast[0] n = ast[1] - collection = node[-3] - list_if = None + # collection = node[-3] + collections = [node[-3]] + list_ifs = [] assert n == 'list_iter' + stores = [] + # Find the list comprehension body. It is the inner-most # node that is not list_.. . while n == 'list_iter': n = n[0] # recurse one step if n == 'list_for': - store = n[2] + stores.append(n[2]) n = n[3] + if self.version >= 3.6 and n[0] == 'list_for': + # Dog-paddle down largely singleton reductions + # to find the collection (expr) + c = n[0][0] + if c == 'expr': + c = c[0] + # FIXME: grammar is wonky here? Is this really an attribute? + if c == 'attribute': + c = c[0] + collections.append(c) + pass elif n in ('list_if', 'list_if_not'): # FIXME: just a guess if n[0].kind == 'expr': - list_if = n + list_ifs.append(n) else: - list_if = n[1] + list_ifs.append([1]) n = n[2] pass pass @@ -1685,12 +1698,24 @@ class SourceWalker(GenericASTTraversal, object): assert n == 'lc_body', ast self.preorder(n[0]) - self.write(' for ') - self.preorder(store) - self.write(' in ') - self.preorder(collection) - if list_if: - self.preorder(list_if) + if self.version < 3.6: + self.write(' for ') + self.preorder(stores[0]) + self.write(' in ') + self.preorder(collections[0]) + if list_ifs: + self.preorder(list_ifs[0]) + pass + else: + for i, store in enumerate(stores): + self.write(' for ') + self.preorder(store) + self.write(' in ') + self.preorder(collections[i]) + if i < len(list_ifs): + self.preorder(list_ifs[i]) + pass + pass self.prec = p def n_listcomp(self, node): @@ -1705,7 +1730,7 @@ class SourceWalker(GenericASTTraversal, object): n_dict_comp = n_set_comp def setcomprehension_walk3(self, node, collection_index): - """List comprehensions the way they are done in Python3. + """Set comprehensions the way they are done in Python3. They're more other comprehensions, e.g. set comprehensions See if we can combine code. """