First list comprehensions from Python3. More test makefile hacking

test/Makefile

@@ -10,10 +10,10 @@ NATIVE_CHECK = check-$(PYTHON_VERSION)
 COMPILE ?=
 
 #: Run working tests from Python 2.7
-check-2.7: check-short-2.7 check-bytecode check-2.7-ok
+check-2.7: check-short-2.7 check-bytecode-2.5 check-2.7-ok
 
 #: Run working tests from Python 3.4
-check-3.4: check-short-3.4 check-short-2.7 check-bytecode
+check-3.4: check-short-3.4 check-short-2.7 check-bytecode-2.5 check-bytecode-3.2
 
 check: check-short
 	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
@@ -27,10 +27,15 @@ check-short:
 check-disasm:
 	$(PYTHON) dis-compare.py
 
-#: Check deparsing only, but from a different Python version
+#: Check deparsing only, from Python 2.5
-check-bytecode:
+check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+
+#: Check deparsing only, from Python 3.2
+check-bytecode-3.2:
+	$(PYTHON) test_pythonlib.py --bytecode-3.2
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)

test/simple-source/comprehension/01-lc.py

@@ -0,0 +1,18 @@
+# Simple list commprehensions
+#
+# Python2 grammar includes:
+#  list_compr ::= BUILD_LIST_0 list_iter
+#  list_iter ::= list_for
+#  list_for ::= expr _for designator list_iter JUMP_BACK
+#  list_iter ::= lc_body
+#  lc_body ::= expr LIST_APPEND
+#
+# Python3 grammar includes:
+#  listcomp ::= LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
+
+[ i for i in (1, 2, 3, 4) ]
+[ i+1 for i in (1, 2, 3, 4) ]
+
+# Try next:
+# [ i * i for i in range(4) ]
+# [ i * j for i in range(4) for j in range(7) ]

test/test_pythonlib.py

@@ -63,6 +63,9 @@ test_options = {
     'bytecode-2.7':
         ['bytecode_2.7', PYC, 'bytecode_2.7', 2.7],
 
+    'bytecode-3.2':
+        ['bytecode_3.2', PYC, 'bytecode_3.2', 3.2],
+
     'bytecode-3.4':
         ['bytecode_3.4', PYC, 'bytecode_3.4', 3.4],
 

uncompyle6/deparser.py

@@ -483,7 +483,7 @@ class Traverser(walker.Walker, object):
 
         n = ast[iter_index]
         assert n == 'comp_iter'
-        # find innerst node
+        # find innermost node
         while n == 'comp_iter':
             n = n[0] # recurse one step
             if   n == 'comp_for':	n = n[3]
@@ -494,14 +494,66 @@ class Traverser(walker.Walker, object):
         self.preorder(n[0])
         self.write(' for ')
         start = len(self.f.getvalue())
-        self.preorder(ast[iter_index-1])
+        designator = ast[iter_index-1]
-        self.set_pos_info(node, start, len(self.f.getvalue()))
+        self.preorder(designator)
+        self.set_pos_info(ast[iter_index-1], start, len(self.f.getvalue()))
         self.write(' in ')
         start = len(self.f.getvalue())
         node[-3].parent = node
         self.preorder(node[-3])
-        self.set_pos_info(node, start, len(self.f.getvalue()))
+        self.set_pos_info(node[-3], start, len(self.f.getvalue()))
+        start = len(self.f.getvalue())
         self.preorder(ast[iter_index])
+        self.set_pos_info(iter_index, start, len(self.f.getvalue()))
+        self.prec = p
+
+    def listcomprehension_walk3(self, node, iter_index, code_index=-5):
+        """List comprehensions the way they are done in Python3.
+        They're more other comprehensions, e.g. set comprehensions
+        See if we can combine code.
+        """
+        p = self.prec
+        self.prec = 27
+        code = node[code_index].attr
+
+        assert inspect.iscode(code)
+        code = Code(code, self.scanner, self.currentclass)
+        # assert isinstance(code, Code)
+
+        ast = self.build_ast(code._tokens, code._customize)
+        self.customize(code._customize)
+        ast = ast[0][0][0][0][0]
+
+        n = ast[iter_index]
+        assert n == 'list_iter'
+        # find innermost node
+        while n == 'list_iter': # list_iter
+            n = n[0] # recurse one step
+            if   n == 'list_for':
+                designator = n[2]
+                n = n[3]
+            elif n == 'list_if':
+                # FIXME: just a guess
+                designator = n[1]
+
+                n = n[2]
+            elif n == 'list_ifnot':
+                # FIXME: just a guess
+                designator = n[1]
+                n = n[2]
+        assert n == 'lc_body', ast
+
+        self.preorder(n[0])
+        self.write(' for ')
+        start = len(self.f.getvalue())
+        self.preorder(designator)
+        self.set_pos_info(designator, start, len(self.f.getvalue()))
+        self.write(' in ')
+        start = len(self.f.getvalue())
+        node[-3].parent = node
+        self.preorder(node[-3])
+        self.set_pos_info(node[-3], start, len(self.f.getvalue()))
+        # self.preorder(ast[iter_index])
         self.prec = p
 
     def n_genexpr(self, node):

uncompyle6/parsers/parse3.py

@@ -54,6 +54,10 @@ class Python3Parser(PythonParser):
 
     def p_list_comprehension(self, args):
         '''
+        expr ::= listcomp
+        listcomp ::= LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
+
+
         expr ::= list_compr
         list_compr ::= BUILD_LIST_0 list_iter
 
@@ -65,7 +69,7 @@ class Python3Parser(PythonParser):
         _come_from ::= COME_FROM
         _come_from ::=
 
-        list_for ::= expr _for designator list_iter JUMP_BACK
+        list_for ::= expr FOR_ITER designator list_iter JUMP_ABSOLUTE
         list_if ::= expr jmp_false list_iter
         list_if_not ::= expr jmp_true list_iter
 
@@ -96,7 +100,7 @@ class Python3Parser(PythonParser):
 
         comp_if ::= expr jmp_false comp_iter
         comp_ifnot ::= expr jmp_true comp_iter
-        comp_for ::= expr _for designator comp_iter JUMP_BACK
+        comp_for ::= expr _for designator comp_iter JUMP_ABSOLUTE
         '''
 
     def p_genexpr(self, args):

uncompyle6/scanners/scanner34.py

@@ -15,7 +15,7 @@ for later use in deparsing.
 
 from __future__ import print_function
 
-import dis
+import dis, inspect
 from collections import namedtuple
 from array import array
 
@@ -108,7 +108,27 @@ class Scanner34(scan.Scanner):
             # keyword attribute names in a call. I suspect there will be things
             # other than LOAD_CONST, but we'll start out with just this for now.
             if opname in ['LOAD_CONST']:
-                pattr = inst.argval
+                const = inst.argval
+                if inspect.iscode(const):
+                    if const.co_name == '<lambda>':
+                        opname = 'LOAD_LAMBDA'
+                    elif const.co_name == '<genexpr>':
+                        opname = 'LOAD_GENEXPR'
+                    elif const.co_name == '<dictcomp>':
+                        opname = 'LOAD_DICTCOMP'
+                    elif const.co_name == '<setcomp>':
+                        opname = 'LOAD_SETCOMP'
+                    elif const.co_name == '<listcomp>':
+                        opname = 'LOAD_LISTCOMP'
+                    # verify() uses 'pattr' for comparison, since 'attr'
+                    # now holds Code(const) and thus can not be used
+                    # for comparison (todo: think about changing this)
+                    # pattr = 'code_object @ 0x%x %s->%s' %\
+                    # (id(const), const.co_filename, const.co_name)
+                    pattr = '<code_object ' + const.co_name + '>'
+                else:
+                    pattr = const
+                    pass
             elif opname in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET', 'BUILD_SLICE',
                             'UNPACK_SEQUENCE',
                             'MAKE_FUNCTION', 'MAKE_CLOSURE',

uncompyle6/walker.py

@@ -910,9 +910,12 @@ class Walker(GenericASTTraversal, object):
         self.prune() # stop recursing
 
     def n_list_compr(self, node):
+        """List comprehensions the way they are done in Python2.
+        """
         p = self.prec
         self.prec = 27
         n = node[-1]
+
         assert n == 'list_iter'
         # find innerst node
         while n == 'list_iter':
@@ -928,10 +931,10 @@ class Walker(GenericASTTraversal, object):
         self.prec = p
         self.prune() # stop recursing
 
-    def comprehension_walk(self, node, iter_index):
+    def comprehension_walk(self, node, iter_index, code_index=-5):
         p = self.prec
         self.prec = 27
-        code = node[-5].attr
+        code = node[code_index].attr
 
         assert inspect.iscode(code)
         code = Code(code, self.scanner, self.currentclass)
@@ -940,11 +943,10 @@ class Walker(GenericASTTraversal, object):
         ast = self.build_ast(code._tokens, code._customize)
         self.customize(code._customize)
         ast = ast[0][0][0]
-
         n = ast[iter_index]
         assert n == 'comp_iter'
         # find innerst node
-        while n == 'comp_iter':
+        while n == 'comp_iter': # list_iter
             n = n[0] # recurse one step
             if   n == 'comp_for':	n = n[3]
             elif n == 'comp_if':	n = n[2]
@@ -961,16 +963,66 @@ class Walker(GenericASTTraversal, object):
 
     def n_genexpr(self, node):
         self.write('(')
-        self.comprehension_walk(node, 3)
+        self.comprehension_walk(node, iter_index=3)
         self.write(')')
         self.prune()
 
     def n_setcomp(self, node):
         self.write('{')
-        self.comprehension_walk(node, 4)
+        self.comprehension_walk(node, iter_index=4)
         self.write('}')
         self.prune()
 
+    def listcomprehension_walk3(self, node, iter_index, code_index=-5):
+        """List comprehensions the way they are done in Python3.
+        They're more other comprehensions, e.g. set comprehensions
+        See if we can combine code.
+        """
+        p = self.prec
+        self.prec = 27
+        code = node[code_index].attr
+
+        assert inspect.iscode(code)
+        code = Code(code, self.scanner, self.currentclass)
+        # assert isinstance(code, Code)
+
+        ast = self.build_ast(code._tokens, code._customize)
+        self.customize(code._customize)
+        ast = ast[0][0][0][0][0]
+
+        n = ast[iter_index]
+        assert n == 'list_iter'
+        # find innermost node
+        while n == 'list_iter': # list_iter
+            n = n[0] # recurse one step
+            if   n == 'list_for':
+                designator = n[2]
+                n = n[3]
+            elif n == 'list_if':
+                # FIXME: just a guess
+                designator = n[1]
+
+                n = n[2]
+            elif n == 'list_ifnot':
+                # FIXME: just a guess
+                designator = n[1]
+                n = n[2]
+        assert n == 'lc_body', ast
+
+        self.preorder(n[0])
+        self.write(' for ')
+        self.preorder(designator)
+        self.write(' in ')
+        self.preorder(node[-3])
+        # self.preorder(ast[iter_index])
+        self.prec = p
+
+    def n_listcomp(self, node):
+        self.write('[')
+        self.listcomprehension_walk3(node, iter_index=1, code_index=0)
+        self.write(']')
+        self.prune()
+
     n_dictcomp = n_setcomp
 
     def n_classdef(self, node):