Python 3 set comprehension bug

test/simple_source/comprehension/05_set_comprehension.py

@@ -8,3 +8,7 @@ b = {v: k for k, v in enumerate(b3)}
 def __new__(classdict):
     members = {k: classdict[k] for k in classdict._member_names}
     return members
+
+# Bug from Python 3.4 asyncio/tasks.py
+def as_completed(fs, *, loop=None):
+    todo = {async(f, loop=loop) for f in set(fs)}

uncompyle6/parsers/parse3.py

@@ -186,8 +186,10 @@ class Python3Parser(PythonParser):
         kwargs  ::=
 
         classdef ::= build_class designator
+
         # Python3 introduced LOAD_BUILD_CLASS
-        # the definition of build_class is a custom rule
+        # Other definitions are in a custom rule
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call_function CALL_FUNCTION_3
 
         stmt ::= classdefdeco
         classdefdeco ::= classdefdeco1 designator
@@ -392,6 +394,9 @@ class Python3Parser(PythonParser):
         assert call_fn_tok, "build_class custom rule needs to find CALL_FUNCTION"
 
         # customize build_class rule
+        # FIXME: What's the deal with the two rules? Different Python versions?
+        # Different situations? Note that the above rule is based on the CALL_FUNCTION
+        # token found, while this one doesn't.
         call_function = self.call_fn_name(call_fn_tok)
         args_pos = call_fn_tok.attr & 0xff
         args_kw = (call_fn_tok.attr >> 8) & 0xff
@@ -399,13 +404,6 @@ class Python3Parser(PythonParser):
                 "%s" % (('expr ' * (args_pos - 1) + ('kwarg ' * args_kw)),
                         call_function))
         self.add_unique_rule(rule, opname, token.attr, customize)
-
-        # FIXME: What's the deal with the two rules? Different Python versions?
-        # Different situations? Note that the above rule is based on the CALL_FUNCTION
-        # token found, while this one doesn't.
-        rule = ("build_class ::= LOAD_BUILD_CLASS mkfunc expr call_function "
-                "CALL_FUNCTION_3")
-        self.add_unique_rule(rule, opname, token.attr, customize)
         return
 
     def custom_classfunc_rule(self, opname, token, customize):
@@ -567,12 +565,17 @@ class Python3Parser(PythonParser):
                                      ('pos_arg ' * args_pos, opname),
                                      opname, token.attr, customize)
                 if self.version >= 3.4:
-                    rule = ('listcomp ::= %sload_closure LOAD_LISTCOMP LOAD_CONST %s expr '
+                    rule1 = ('listcomp ::= %sload_closure LOAD_LISTCOMP LOAD_CONST %s expr '
+                            'GET_ITER CALL_FUNCTION_1' % ('expr ' * args_pos, opname))
+                    rule2 = ('setcomp ::= %sload_closure LOAD_SETCOMP LOAD_CONST %s expr '
                             'GET_ITER CALL_FUNCTION_1' % ('expr ' * args_pos, opname))
                 else:
-                    rule = ('listcomp ::= %sload_closure LOAD_LISTCOMP %s expr '
+                    rule1 = ('listcomp ::= %sload_closure LOAD_LISTCOMP %s expr '
                             'GET_ITER CALL_FUNCTION_1' % ('expr ' * args_pos, opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+                    rule2 = ('setcomp ::= %sload_closure LOAD_SETCOMP %s expr '
+                            'GET_ITER CALL_FUNCTION_1' % ('expr ' * args_pos, opname))
+                self.add_unique_rule(rule1, opname, token.attr, customize)
+                self.add_unique_rule(rule2, opname, token.attr, customize)
 
                 self.add_unique_rule('dictcomp ::= %sload_closure LOAD_DICTCOMP %s '
                                      'expr GET_ITER CALL_FUNCTION_1' %

uncompyle6/semantics/fragments.py

@@ -703,6 +703,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
             start = len(self.f.getvalue())
             self.set_pos_info(node[0], start-1, start)
             self.listcomprehension_walk3(node, 1, 0)
+        elif node[0].type == 'load_closure':
+            self.setcomprehension_walk3(node, collection_index=4)
         else:
             self.comprehension_walk(node, iter_index=4)
         self.write('}')
@@ -721,6 +723,59 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.write(']')
         self.prune()
 
+    def setcomprehension_walk3(self, node, collection_index):
+        """List comprehensions the way they are done in Python3.
+        They're more other comprehensions, e.g. set comprehensions
+        See if we can combine code.
+        """
+        p = self.prec
+        self.prec = 27
+
+        code = Code(node[1].attr, self.scanner, self.currentclass)
+        ast = self.build_ast(code._tokens, code._customize)
+        self.customize(code._customize)
+        ast = ast[0][0][0]
+        designator = ast[3]
+        collection = node[collection_index]
+
+        n = ast[4]
+        list_if = None
+        assert n == 'comp_iter'
+
+        # find innermost node
+        while n == 'comp_iter':
+            n = n[0] # recurse one step
+            # FIXME: adjust for set comprehension
+            if n == 'list_for':
+                designator = n[2]
+                n = n[3]
+            elif n in ('list_if', 'list_if_not'):
+                # FIXME: just a guess
+                if n[0].type == 'expr':
+                    list_if = n
+                else:
+                    list_if = n[1]
+                n = n[2]
+                pass
+            pass
+
+        assert n == 'comp_body', ast
+
+        self.preorder(n[0])
+        self.write(' for ')
+        start = len(self.f.getvalue())
+        self.preorder(designator)
+        self.set_pos_info(designator, start, len(self.f.getvalue()))
+        self.write(' in ')
+        start = len(self.f.getvalue())
+        self.preorder(collection)
+        self.set_pos_info(collection, start, len(self.f.getvalue()))
+        if list_if:
+            start = len(self.f.getvalue())
+            self.preorder(list_if)
+            self.set_pos_info(list_if, start, len(self.f.getvalue()))
+        self.prec = p
+
     def n_classdef(self, node):
         # class definition ('class X(A,B,C):')
         cclass = self.currentclass

uncompyle6/semantics/pysource.py

@@ -1028,7 +1028,10 @@ class SourceWalker(GenericASTTraversal, object):
             else:
                 assert False, "Can't find code for comprehension"
 
-        assert iscode(cn.attr)
+        try:
+            assert iscode(cn.attr)
+        except:
+            from trepan.api import debug; debug()
         code = Code(cn.attr, self.scanner, self.currentclass)
         ast = self.build_ast(code._tokens, code._customize)
         self.customize(code._customize)
@@ -1063,6 +1066,8 @@ class SourceWalker(GenericASTTraversal, object):
         self.write('{')
         if node[0] in ['LOAD_SETCOMP', 'LOAD_DICTCOMP']:
             self.listcomprehension_walk3(node, 1, 0)
+        elif node[0].type == 'load_closure':
+            self.setcomprehension_walk3(node, collection_index=4)
         else:
             self.comprehension_walk(node, iter_index=4)
         self.write('}')
@@ -1132,7 +1137,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec = p
 
     def listcomprehension_walk2(self, node):
-        """List comprehensions the way they are done in Python3.
+        """List comprehensions the way they are done in Python 2.
         They're more other comprehensions, e.g. set comprehensions
         See if we can combine code.
         """
@@ -1142,7 +1147,10 @@ class SourceWalker(GenericASTTraversal, object):
         code = Code(node[1].attr, self.scanner, self.currentclass)
         ast = self.build_ast(code._tokens, code._customize)
         self.customize(code._customize)
-        ast = ast[0][0][0][0][0]
+        if node == 'setcomp':
+            ast = ast[0][0][0]
+        else:
+            ast = ast[0][0][0][0][0]
 
         n = ast[1]
         collection = node[-3]
@@ -1152,7 +1160,7 @@ class SourceWalker(GenericASTTraversal, object):
         # find innermost node
         while n == 'list_iter':
             n = n[0] # recurse one step
-            if   n == 'list_for':
+            if n == 'list_for':
                 designator = n[2]
                 n = n[3]
             elif n in ('list_if', 'list_if_not'):
@@ -1187,6 +1195,53 @@ class SourceWalker(GenericASTTraversal, object):
 
     n_dictcomp = n_setcomp
 
+    def setcomprehension_walk3(self, node, collection_index):
+        """List comprehensions the way they are done in Python3.
+        They're more other comprehensions, e.g. set comprehensions
+        See if we can combine code.
+        """
+        p = self.prec
+        self.prec = 27
+
+        code = Code(node[1].attr, self.scanner, self.currentclass)
+        ast = self.build_ast(code._tokens, code._customize)
+        self.customize(code._customize)
+        ast = ast[0][0][0]
+        designator = ast[3]
+        collection = node[collection_index]
+
+        n = ast[4]
+        list_if = None
+        assert n == 'comp_iter'
+
+        # find innermost node
+        while n == 'comp_iter':
+            n = n[0] # recurse one step
+            # FIXME: adjust for set comprehension
+            if n == 'list_for':
+                designator = n[2]
+                n = n[3]
+            elif n in ('list_if', 'list_if_not'):
+                # FIXME: just a guess
+                if n[0].type == 'expr':
+                    list_if = n
+                else:
+                    list_if = n[1]
+                n = n[2]
+                pass
+            pass
+
+        assert n == 'comp_body', ast
+
+        self.preorder(n[0])
+        self.write(' for ')
+        self.preorder(designator)
+        self.write(' in ')
+        self.preorder(collection)
+        if list_if:
+            self.preorder(list_if)
+        self.prec = p
+
     def n_classdef(self, node):
         # class definition ('class X(A,B,C):')
         cclass = self.currentclass