Expand annotate handling to 3.3

(and possibly 3.2)

- DRY Python 3.1-3.3 grammar a little

pytest/test_grammar.py

@@ -1,4 +1,4 @@
-import pytest, re
+import re
 from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser
 from uncompyle6.scanner import get_scanner
@@ -16,7 +16,8 @@ def test_grammar():
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
     lhs, rhs, tokens, right_recursive = p.checkSets()
     expect_lhs = set(['expr1024', 'pos_arg'])
-    unused_rhs = set(['build_list', 'call_function', 'mkfunc', 'mklambda',
+    unused_rhs = set(['build_list', 'call_function', 'mkfunc',
+                      'mklambda',
                       'unpack', 'unpack_list'])
     expect_right_recursive = [['designList', ('designator', 'DUP_TOP', 'designList')]]
     if PYTHON3:
@@ -24,6 +25,9 @@ def test_grammar():
         unused_rhs = unused_rhs.union(set("""
         except_pop_except genexpr classdefdeco2 listcomp
         """.split()))
+        if 3.1 <= PYTHON_VERSION <= 3.3:
+            unused_rhs.add("mkfunc_annotate")
+            pass
     else:
         expect_lhs.add('kwarg')
     assert expect_lhs == set(lhs)

uncompyle6/parsers/parse31.py

@@ -32,9 +32,6 @@ class Python31Parser(Python32Parser):
         store ::= STORE_NAME
         load  ::= LOAD_FAST
         load  ::= LOAD_NAME
-
-        stmt ::= funcdef_annotate
-        funcdef_annotate ::= mkfunc_annotate designator
         """
 
     def add_custom_rules(self, tokens, customize):
@@ -48,8 +45,7 @@ class Python31Parser(Python32Parser):
                 #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
                 rule = ('mkfunc_annotate ::= %s%sLOAD_CONST EXTENDED_ARG %s' %
                         (('pos_arg ' * (args_pos)),
-                         ('kwargs ' * (annotate_args-1)), opname))
+                         ('annotate_args ' * (annotate_args-1)), opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
-
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse32.py

@@ -14,8 +14,29 @@ class Python32Parser(Python3Parser):
         binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
         stmt ::= store_locals
         store_locals ::= LOAD_FAST STORE_LOCALS
+
+        stmt ::= funcdef_annotate
+        funcdef_annotate ::= mkfunc_annotate designator
+
+        annotate_args   ::= annotate_args annotate_arg
+        annotate_args   ::= annotate_arg
+        annotate_arg    ::= LOAD_CONST expr
         """
     pass
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python32Parser, self).add_custom_rules(tokens, customize)
+        for i, token in enumerate(tokens):
+            opname = token.type
+            if opname.startswith('MAKE_FUNCTION_A'):
+                args_pos, args_kw, annotate_args  = token.attr
+                # Check that there are 2 annotated params?
+                rule = ('mkfunc_annotate ::= %s%sLOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                        (('pos_arg ' * (args_pos)),
+                         ('annotate_args ' * (annotate_args-1)), opname))
+                print(rule)
+                self.add_unique_rule(rule, opname, token.attr, customize)
+
+
 class Python32ParserSingle(Python32Parser, PythonParserSingle):
     pass

uncompyle6/semantics/pysource.py

@@ -612,15 +612,16 @@ class SourceWalker(GenericASTTraversal, object):
             })
 
         if version >= 3.0:
-            if 3.1 == version:
+            if 3.1 <= version <= 3.3:
                 ##########################
-                # Python 3.1
+                # Python 3.1 - 3.3
                 ##########################
                 TABLE_DIRECT.update({
                     'funcdef_annotate': ( '\n\n%|def %c%c\n', -1, 0),
+                    'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
                     })
 
-                def make_function31(node, isLambda, nested=1,
+                def make_function3(node, isLambda, nested=1,
                                    codeNode=None, annotate=None):
                     """Dump function defintion, doc string, and function
                     body. This code is specialzed for Python 3.1"""
@@ -653,7 +654,12 @@ class SourceWalker(GenericASTTraversal, object):
                         kw_args  = 0
                         pass
 
+                    if 3.0 <= self.version <= 3.2:
                         lambda_index = -2
+                    elif 3.03 <= self.version:
+                        lambda_index = -3
+                    else:
+                        lambda_index = None
 
                     if lambda_index and isLambda and iscode(node[lambda_index].attr):
                         assert node[lambda_index].type == 'LOAD_LAMBDA'
@@ -757,18 +763,30 @@ class SourceWalker(GenericASTTraversal, object):
                                     returnNone=rn)
                     code._tokens = code._customize = None # save memory
 
-                self.make_function31 = make_function31
+                self.make_function3 = make_function3
 
                 def n_mkfunc_annotate(node):
 
-                    self.indentMore()
+                    if self.version >= 3.3 or node[-2] == 'kwargs':
+                        # LOAD_CONST code object ..
+                        # LOAD_CONST        'x0'  if >= 3.3
+                        # EXTENDED_ARG
+                        # MAKE_FUNCTION ..
+                        code = node[-4]
+                    elif node[-3] == 'expr':
+                        code = node[-3][0]
+                    else:
+                        # LOAD_CONST code object ..
+                        # MAKE_FUNCTION ..
                         code = node[-3]
+
+                    self.indentMore()
                     annotate = None
-                    if node[-4] == 'kwargs' and node[-4][0][0] == 'LOAD_CONST':
+                    annotate_args = node[-4] if self.version == 3.1 else node[-5];
-                        # FIXME: Annotate args are currently stored as kwargs
+
-                        # we should rename that grammar nonterminal
+                    if annotate_args == 'annotate_args' and annotate_args[0][0] == 'LOAD_CONST':
-                        annotate = node[-4][0][0].attr
+                        annotate = annotate_args[0][0].attr
-                    self.make_function31(node, isLambda=False,
+                    self.make_function3(node, isLambda=False,
                                         codeNode=code, annotate=annotate)
 
                     if len(self.param_stack) > 1:
@@ -780,13 +798,6 @@ class SourceWalker(GenericASTTraversal, object):
                 self.n_mkfunc_annotate = n_mkfunc_annotate
 
 
-            elif 3.2 <= version <= 3.3:
-                ##########################
-                # Python 3.2 and 3.3
-                ##########################
-                TABLE_DIRECT.update({
-                    'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
-                    })
             elif version >= 3.4:
                 ########################
                 # Python 3.4+ Additions