Merge pull request #259 from rocky/annotation-types-final

Fix py3 function signatures + annotations + ordering

test/simple_source/bug31/04_def_annotate.py

@@ -47,11 +47,50 @@ def div(a: dict(type=float, help='the dividend'),
     """Divide a by b"""
     return a / b
 
-# FIXME:
+class TestSignatureObject1():
-# class TestSignatureObject():
+    def test_signature_on_wkwonly(self):
-#     def test_signature_on_wkwonly(self):
+        def test(*, a:float, b:str, c:str = 'test', **kwargs: int) -> int:
-#         def test(*, a:float, b:str) -> int:
+            pass
-#             pass
+
+class TestSignatureObject2():
+    def test_signature_on_wkwonly(self):
+        def test(*, c='test', a:float, b:str="S", **kwargs: int) -> int:
+            pass
+
+class TestSignatureObject3():
+    def test_signature_on_wkwonly(self):
+        def test(*, c='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject4():
+    def test_signature_on_wkwonly(self):
+        def test(x=55, *args, c:str='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject5():
+    def test_signature_on_wkwonly(self):
+        def test(x=55, *args: int, c='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject5():
+    def test_signature_on_wkwonly(self):
+        def test(x:int=55, *args: (int, str), c='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject7():
+    def test_signature_on_wkwonly(self):
+        def test(c='test', kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject8():
+    def test_signature_on_wkwonly(self):
+        def test(**b: int) -> int:
+            pass
+
+class TestSignatureObject9():
+    def test_signature_on_wkwonly(self):
+        def test(a, **b: int) -> int:
+            pass
 
 class SupportsInt():
 

uncompyle6/parsers/parse3.py

@@ -1080,11 +1080,11 @@ class Python3Parser(PythonParser):
                     else:
                         # See above comment about use of EXTENDED_ARG
                         rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                                (('pos_arg ' * (args_pos)),  ('kwargs ' * args_kw),
+                                (('kwargs ' * args_kw), ('pos_arg ' * (args_pos)),
                                  ('annotate_arg ' * (annotate_args-1)), opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                                (('pos_arg ' * (args_pos)),  ('kwargs ' * args_kw),
+                                (('kwargs ' * args_kw), ('pos_arg ' * (args_pos)),
                                  ('call ' * (annotate_args-1)), opname))
                     self.addRule(rule, nop_func)
             elif opname == 'RETURN_VALUE_LAMBDA':

uncompyle6/semantics/make_function.py

@@ -91,6 +91,12 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
         annotate_argc = 0
         pass
 
+    annotate_dict = {}
+
+    for name in annotate_args.keys():
+        n = self.traverse(annotate_args[name], indent='')
+        annotate_dict[name] = n
+
     if 3.0 <= self.version <= 3.2:
         lambda_index = -2
     elif 3.03 <= self.version:
@@ -109,7 +115,11 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
 
     # add defaults values to parameter names
     argc = code.co_argcount
+    kwonlyargcount = code.co_kwonlyargcount
+
     paramnames = list(code.co_varnames[:argc])
+    if kwonlyargcount > 0:
+        kwargs = list(code.co_varnames[argc:argc+kwonlyargcount])
 
     try:
         ast = self.build_ast(code._tokens,
@@ -135,10 +145,6 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
     indent = ' ' * l
     line_number = self.line_number
 
-    if code_has_star_arg(code):
-        self.write('*%s' % code.co_varnames[argc + kw_pairs])
-        argc += 1
-
     i = len(paramnames) - len(defparams)
     suffix = ''
 
@@ -147,10 +153,8 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
     for param in paramnames[:i]:
         self.write(suffix, param)
         suffix = ', '
-        if param in annotate_tuple[0].attr:
+        if param in annotate_dict:
-            p = annotate_tuple[0].attr.index(param)
+            self.write(': %s' % annotate_dict[param])
-            self.write(': ')
-            self.preorder(node[p])
             if (line_number != self.line_number):
                 suffix = ",\n" + indent
                 line_number = self.line_number
@@ -187,8 +191,17 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
                 suffix = ', '
 
 
+    if code_has_star_arg(code):
+        star_arg = code.co_varnames[argc + kwonlyargcount]
+        if annotate_dict and star_arg in annotate_dict:
+            self.write(suffix, '*%s: %s' % (star_arg, annotate_dict[star_arg]))
+        else:
+            self.write(suffix, '*%s' % star_arg)
+        argc += 1
+
     # self.println(indent, '#flags:\t', int(code.co_flags))
-    if kw_args + annotate_argc > 0:
+    ends_in_comma = False
+    if kwonlyargcount > 0:
         if no_paramnames:
             if not code_has_star_arg(code):
                 if argc > 0:
@@ -198,49 +211,52 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
                 pass
             else:
                 self.write(", ")
-
+            ends_in_comma = True
-            kwargs = node[0]
+        else:
-            last = len(kwargs)-1
-            i = 0
-            for n in node[0]:
-                if n == 'kwarg':
-                    if (line_number != self.line_number):
-                        self.write("\n" + indent)
-                        line_number = self.line_number
-                    self.write('%s=' % n[0].pattr)
-                    self.preorder(n[1])
-                    if i < last:
-                        self.write(', ')
-                    i += 1
-                    pass
-                pass
-            annotate_args = []
-            for n in node:
-                if n == 'annotate_arg':
-                    annotate_args.append(n[0])
-                elif n == 'annotate_tuple':
-                    t = n[0].attr
-                    if t[-1] == 'return':
-                        t = t[0:-1]
-                        annotate_args = annotate_args[:-1]
-                        pass
-                    last = len(annotate_args) - 1
-                    for i in range(len(annotate_args)):
-                        self.write("%s: " % (t[i]))
-                        self.preorder(annotate_args[i])
-                        if i < last:
-                            self.write(', ')
-                            pass
-                        pass
-                    break
-                pass
-            pass
-
-
-        if code_has_star_star_arg(code):
             if argc > 0:
                 self.write(', ')
-            self.write('**%s' % code.co_varnames[argc + kw_pairs])
+                ends_in_comma = True
+
+        kw_args = [None] * kwonlyargcount
+
+        for n in node:
+            if n == 'kwargs':
+                n = n[0]
+            if n == 'kwarg':
+                name = eval(n[0].pattr)
+                idx = kwargs.index(name)
+                default = self.traverse(n[1], indent='')
+                if annotate_dict and name in annotate_dict:
+                    kw_args[idx] = '%s: %s=%s' % (name, annotate_dict[name], default)
+                else:
+                    kw_args[idx] = '%s=%s' % (name, default)
+                pass
+            pass
+
+        # handling other args
+        ann_other_kw = [c == None for c in kw_args]
+        for i, flag in enumerate(ann_other_kw):
+            if flag:
+                n = kwargs[i]
+                if n in annotate_dict:
+                    kw_args[i] = "%s: %s" %(n, annotate_dict[n])
+                else:
+                    kw_args[i] = "%s" % n
+
+        self.write(', '.join(kw_args), ', ')
+
+    else:
+        if argc == 0:
+            ends_in_comma = True
+
+    if code_has_star_star_arg(code):
+        if not ends_in_comma:
+            self.write(', ')
+        star_star_arg = code.co_varnames[argc + kwonlyargcount]
+        if annotate_dict and star_star_arg in annotate_dict:
+            self.write('**%s: %s' % (star_star_arg, annotate_dict[star_star_arg]))
+        else:
+            self.write('**%s' % star_star_arg)
 
     if is_lambda:
         self.write(": ")
@@ -476,7 +492,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     # Thank you, Python.
 
-    def build_param(ast, name, default):
+    def build_param(ast, name, default, annotation=None):
         """build parameters:
             - handle defaults
             - handle format tuple parameters
@@ -486,6 +502,9 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
         else:
             value = self.traverse(default, indent='')
         maybe_show_tree_param_default(self.showast, name, value)
+        if annotation:
+            result = '%s: %s=%s' % (name, annotation, value)
+        else:
             result = '%s=%s' % (name, value)
 
         # The below can probably be removed. This is probably
@@ -654,7 +673,11 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     # add defaults values to parameter names
     argc = code.co_argcount
+    kwonlyargcount = code.co_kwonlyargcount
+
     paramnames = list(scanner_code.co_varnames[:argc])
+    if kwonlyargcount > 0:
+        kwargs = list(scanner_code.co_varnames[argc:argc+kwonlyargcount])
 
     # defaults are for last n parameters, thus reverse
     paramnames.reverse();
@@ -677,21 +700,37 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     else:
         kw_pairs = 0
 
+    i = len(paramnames) - len(defparams)
+    no_paramnames = len(paramnames[:i]) == 0
+
     # build parameters
     params = []
     if defparams:
         for i, defparam in enumerate(defparams):
-            params.append(build_param(ast, paramnames[i], defparam))
+            params.append(build_param(ast, paramnames[i], defparam,
+                                      annotate_dict.get(paramnames[i])))
 
-        params += paramnames[i+1:]
+        for param in paramnames[i+1:]:
+            if param in annotate_dict:
+                params.append("%s: %s" % (param, annotate_dict[param]))
             else:
-        params = paramnames
+                params.append(param)
+    else:
+        for param in paramnames:
+            if param in annotate_dict:
+                params.append("%s: %s" % (param, annotate_dict[param]))
+            else:
+                params.append(param)
 
     params.reverse() # back to correct order
 
     if code_has_star_arg(code):
         if self.version > 3.0:
-            params.append('*%s' % code.co_varnames[argc + kw_pairs])
+            star_arg = code.co_varnames[argc + kwonlyargcount]
+            if annotate_dict and star_arg in annotate_dict:
+                params.append('*%s: %s' % (star_arg, annotate_dict[star_arg]))
+            else:
+                params.append('*%s' % star_arg)
         else:
             params.append('*%s' % code.co_varnames[argc])
         argc += 1
@@ -720,10 +759,12 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
         self.write("(", ", ".join(params))
     # self.println(indent, '#flags:\t', int(code.co_flags))
 
-    kwonlyargcount = 0
+    # FIXME: Could we remove ends_in_comma and its tests if we just
-
+    # created a parameter list and at the very end did a join on that?
+    # Unless careful, We might lose line breaks though.
     ends_in_comma = False
-    if kw_args + annotate_argc > 0:
+    if kwonlyargcount > 0:
+        if no_paramnames:
             if not (4 & code.co_flags):
                 if argc > 0:
                     self.write(", *, ")
@@ -733,6 +774,10 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             else:
                 self.write(", ")
                 ends_in_comma = True
+        else:
+            if argc > 0:
+                self.write(', ')
+                ends_in_comma = True
 
         # FIXME: this is not correct for 3.5. or 3.6 (which works different)
         # and 3.7?
@@ -742,7 +787,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             i = 0
             for n in node[0]:
                 if n == 'kwarg':
-                    self.write('%s=' % n[0].pattr)
+                    self.write('%s=' % n[0].attr)
                     self.preorder(n[1])
                     if i < last:
                         self.write(', ')
@@ -767,9 +812,9 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 break
         elif self.version >= 3.6:
             # argc = node[-1].attr
-            co = node[-3].attr
+            # co = node[-3].attr
             # argcount = co.co_argcount
-            kwcount = kwonlyargcount = co.co_kwonlyargcount
+            # kwonlyargcount = co.co_kwonlyargcount
 
             free_tup = ann_dict = kw_dict = default_tup = None
             fn_bits = node[-1].attr
@@ -793,6 +838,8 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 kw_dict = kw_dict[0]
 
             # FIXME: handle free_tup, annotate_dict, and default_tup
+            kw_args = [None] * kwonlyargcount
+
             if kw_dict:
                 assert kw_dict == 'dict'
                 defaults = [self.traverse(n, indent='') for n in kw_dict[:-2]]
@@ -801,33 +848,33 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 sep = ''
                 # FIXME: possibly handle line breaks
                 for i, n in enumerate(names):
-                    self.write(sep)
+                    idx = kwargs.index(n)
                     if annotate_dict and n in annotate_dict:
-                        self.write("%s: %s=%s" % (n, annotate_dict[n], defaults[i]))
+                        t = "%s: %s=%s" % (n, annotate_dict[n], defaults[i])
                     else:
-                        self.write("%s=%s" % (n, defaults[i]))
+                        t = "%s=%s" % (n, defaults[i])
-                    sep = ', '
+                    kw_args[idx] = t
-                    ends_in_comma = False
-                    kwcount -= 1
                     pass
                 pass
-            if ann_dict:
-                annotate_args = ann_dict[0][-2].attr
-                if annotate_args[-1] == 'return':
-                    annotate_args = annotate_args[:-1]
 
-                sep = ''
+            # handle others
-                if not ends_in_comma:
+            if ann_dict:
-                    sep = ', '
+                ann_other_kw = [c == None for c in kw_args]
-                for n in annotate_args:
+
-                    if kwcount == 0:
+                for i, flag in enumerate(ann_other_kw):
-                        break
+                    if flag:
-                    self.write(sep)
+                        n = kwargs[i]
-                    self.write('%s: %s' %(n, annotate_dict[n]))
+                        if n in annotate_dict:
-                    sep = ', '
+                            kw_args[i] = "%s: %s" %(n, annotate_dict[n])
+                        else:
+                            kw_args[i] = "%s" % n
+            self.write(', '.join(kw_args))
             ends_in_comma = False
-                    kwcount -= 1
+
         pass
+    else:
+        if argc == 0:
+            ends_in_comma = True
 
     if code_has_star_star_arg(code):
         if not ends_in_comma: