Fix 3.6+ nested list comprehensions

Modified "load_closure" rule and
changed semantic actions since LOAD_CLOSUREs are stored
inside a MAKE_TUPLE.

FIXME: Not sure if we have additional "if" clauses correct. Test and correct

test/simple_source/bug36/06_listcomp_nest.py

@@ -0,0 +1,18 @@
+# From 3.6 _sitebuiltins.py
+# Bug was in handling double nested kinds of things like:
+#     for a in b for c in d
+
+# This required grammar modification and
+# and semantic action changes. LOAD_CLOSUREs are stored
+# inside a MAKE_TUPLE.
+
+# FIXME: test and try additional "if" clauses.
+def __init__(self, path, name, files=(), dirs=(), volumes=()):
+    f = [path.join(dir, filename)
+        for dir in dirs
+        for filename in files]
+    f2 = [path.join(drive, dir, filename)
+         for dir in dirs
+         for filename in files
+         for drive in volumes]
+    return f, f2

uncompyle6/parsers/parse3.py

@@ -885,8 +885,13 @@ class Python3Parser(PythonParser):
                                    "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))
                         self.add_make_function_rule(rule_pat, opname, token.attr, customize)
                         if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LISTCOMP'):
-                            rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            if self.version < 3.6:
+                                rule_pat = ("listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
                                        "GET_ITER CALL_FUNCTION_1" % ('expr ' * args_pos, opname))
+                            else:
+                                # 3.6+ bundles all of the 'exprs' in the rule above into a tuple.
+                                rule_pat = ("listcomp ::= load_closure LOAD_LISTCOMP %%s%s expr "
+                                        "GET_ITER CALL_FUNCTION_1" % (opname,))
                             self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
                     if is_pypy or (i >= 2 and tokens[i-2] == 'LOAD_LAMBDA'):

uncompyle6/semantics/pysource.py

@@ -1532,10 +1532,9 @@ class SourceWalker(GenericASTTraversal, object):
         self.prune()
 
     def comprehension_walk3(self, node, iter_index, code_index=-5):
-        """
+        """Non-closure-based comprehensions the way they are done in Python3.
-        List comprehensions the way they are done in Python3.
+        They are other comprehensions, e.g. set comprehensions See if
-        They are other comprehensions, e.g. set comprehensions
+        we can combine code.
-        See if we can combine code.
         """
         p = self.prec
         self.prec = 27
@@ -1600,7 +1599,7 @@ class SourceWalker(GenericASTTraversal, object):
         assert n.kind in ('lc_body', 'comp_body', 'setcomp_func', 'set_comp_body'), ast
         assert store, "Couldn't find store in list/set comprehension"
 
-        # Issue created with later Python code generation is that there
+        # A problem created with later Python code generation is that there
         # is a lamda set up with a dummy argument name that is then called
         # So we can't just translate that as is but need to replace the
         # dummy name. Below we are picking out the variable name as seen
@@ -1641,8 +1640,8 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec = p
 
     def listcomprehension_walk2(self, node):
-        """List comprehensions the way they are done in Python 2 (and
+        """List comprehensions the way they are done in Python 2 and
-        some Python 3?).
+        sometimes in Python 3.
         They're more other comprehensions, e.g. set comprehensions
         See if we can combine code.
         """
@@ -1652,32 +1651,46 @@ class SourceWalker(GenericASTTraversal, object):
         code = Code(node[1].attr, self.scanner, self.currentclass)
         ast = self.build_ast(code._tokens, code._customize)
         self.customize(code._customize)
-        if node == 'set_comp':
-            ast = ast[0][0][0]
-        else:
-            ast = ast[0][0][0][0][0]
 
-        if ast == 'expr':
+        # skip over: sstmt, stmt, return, ret_expr
+        # and other singleton derivations
+        while (len(ast) == 1
+               or (ast in ('sstmt', 'return')
+                   and ast[-1] in ('RETURN_LAST', 'RETURN_VALUE'))):
             ast = ast[0]
 
         n = ast[1]
-        collection = node[-3]
+        # collection = node[-3]
-        list_if = None
+        collections = [node[-3]]
+        list_ifs = []
         assert n == 'list_iter'
 
+        stores = []
+
         # Find the list comprehension body. It is the inner-most
         # node that is not list_.. .
         while n == 'list_iter':
             n = n[0] # recurse one step
             if n == 'list_for':
-                store = n[2]
+                stores.append(n[2])
                 n = n[3]
+                if self.version >= 3.6 and n[0] == 'list_for':
+                    # Dog-paddle down largely singleton reductions
+                    # to find the collection (expr)
+                    c = n[0][0]
+                    if c == 'expr':
+                        c = c[0]
+                    # FIXME: grammar is wonky here? Is this really an attribute?
+                    if c == 'attribute':
+                        c = c[0]
+                    collections.append(c)
+                    pass
             elif n in ('list_if', 'list_if_not'):
                 # FIXME: just a guess
                 if n[0].kind == 'expr':
-                    list_if = n
+                    list_ifs.append(n)
                 else:
-                    list_if = n[1]
+                    list_ifs.append([1])
                 n = n[2]
                 pass
             pass
@@ -1685,12 +1698,24 @@ class SourceWalker(GenericASTTraversal, object):
         assert n == 'lc_body', ast
 
         self.preorder(n[0])
-        self.write(' for ')
+        if self.version < 3.6:
-        self.preorder(store)
+            self.write(' for ')
-        self.write(' in ')
+            self.preorder(stores[0])
-        self.preorder(collection)
+            self.write(' in ')
-        if list_if:
+            self.preorder(collections[0])
-            self.preorder(list_if)
+            if list_ifs:
+                self.preorder(list_ifs[0])
+                pass
+        else:
+            for i, store in enumerate(stores):
+                self.write(' for ')
+                self.preorder(store)
+                self.write(' in ')
+                self.preorder(collections[i])
+                if i < len(list_ifs):
+                    self.preorder(list_ifs[i])
+                    pass
+                pass
         self.prec = p
 
     def n_listcomp(self, node):
@@ -1705,7 +1730,7 @@ class SourceWalker(GenericASTTraversal, object):
     n_dict_comp = n_set_comp
 
     def setcomprehension_walk3(self, node, collection_index):
-        """List comprehensions the way they are done in Python3.
+        """Set comprehensions the way they are done in Python3.
         They're more other comprehensions, e.g. set comprehensions
         See if we can combine code.
         """