Merge pull request #117 from rocky/3.6-MAKE_FUNCTION

3.6 make function

README.rst

@@ -168,7 +168,7 @@ See Also
 --------
 
 * https://github.com/zrax/pycdc : supports all versions of Python and is written in C++. Support for later Python 3 versions is a bit lacking though.
-* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique what is used here.
+* https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Include some fixes like supporting function annotations
 * The HISTORY_ file.
 

uncompyle6/parsers/parse3.py

@@ -334,7 +334,9 @@ class Python3Parser(PythonParser):
 
     def p_stmt3(self, args):
         """
-        stmt ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST
+        stmt ::= return_closure
+        return_closure ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST
+
         stmt ::= whileTruestmt
         ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_from
         """
@@ -701,7 +703,22 @@ class Python3Parser(PythonParser):
                 rule = 'unpack_list ::= ' + opname + ' designator' * token.attr
             elif opname_base.startswith('MAKE_FUNCTION'):
                 # DRY with MAKE_CLOSURE
-                args_pos, args_kw, annotate_args  = token.attr
+                if self.version >= 3.6:
+                    # The semantics of MAKE_FUNCTION in 3.6 are totally different from
+                    # before.
+                    args_pos, args_kw, annotate_args, closure  = token.attr
+                    stack_count = args_pos + args_kw + annotate_args
+                    rule = ('mkfunc ::= %s%s%s%s' %
+                                ('expr ' * stack_count,
+                                 'load_closure ' * closure,
+                                 'LOAD_CONST ' * 2,
+                                 opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    continue
+                if self.version < 3.6:
+                    args_pos, args_kw, annotate_args  = token.attr
+                else:
+                    args_pos, args_kw, annotate_args, closure  = token.attr
 
                 rule_pat = ("genexpr ::= %sload_genexpr %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ('pos_arg '* args_pos, opname))

uncompyle6/parsers/parse36.py

@@ -16,6 +16,8 @@ class Python36Parser(Python35Parser):
 
     def p_36misc(self, args):
         """
+        return_closure ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
+
         expr ::= LOAD_NAME EXTENDED_ARG
 
         fstring_multi ::= fstring_expr_or_strs BUILD_STRING

uncompyle6/scanners/scanner3.py

@@ -130,6 +130,9 @@ class Scanner3(Scanner):
             varargs_ops.add(self.opc.CALL_METHOD)
         if self.version >= 3.6:
             varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
+            # Below is in bit order, "default = bit 0, closure = bit 3
+            self.MAKE_FUNCTION_FLAGS = tuple("""
+             default keyword-only annotation closure""".split())
 
         self.varargs_ops = frozenset(varargs_ops)
         # FIXME: remove the above in favor of:
@@ -271,20 +274,37 @@ class Scanner3(Scanner):
                     pattr = const
                     pass
             elif opname in ('MAKE_FUNCTION', 'MAKE_CLOSURE'):
-                pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
+                if self.version >= 3.6:
-                if name_pair_args > 0:
+                    # 3.6+ doesn't have MAKE_CLOSURE, so opname == 'MAKE_CLOSURE'
-                    opname = '%s_N%d' % (opname, name_pair_args)
+                    flags = inst.argval
-                    pass
+                    opname = 'MAKE_FUNCTION_%d' % (flags)
-                if annotate_args > 0:
+                    attr = []
-                    opname = '%s_A_%d' % (opname, annotate_args)
+                    for flag in self.MAKE_FUNCTION_FLAGS:
-                    pass
+                        bit = flags & 1
-                opname = '%s_%d' % (opname, pos_args)
+                        if bit:
-                pattr = ("%d positional, %d keyword pair, %d annotated" %
+                            if pattr:
-                             (pos_args, name_pair_args, annotate_args))
+                                pattr += ", " + flag
+                            else:
+                                pattr += flag
+                        attr.append(bit)
+                        flags >>= 1
+                    attr = attr[:4] # remove last value: attr[5] == False
+                else:
+                    pos_args, name_pair_args, annotate_args = parse_fn_counts(inst.argval)
+                    pattr = ("%d positional, %d keyword pair, %d annotated" %
+                                 (pos_args, name_pair_args, annotate_args))
+                    if name_pair_args > 0:
+                        opname = '%s_N%d' % (opname, name_pair_args)
+                        pass
+                    if annotate_args > 0:
+                        opname = '%s_A_%d' % (opname, annotate_args)
+                        pass
+                    opname = '%s_%d' % (opname, pos_args)
+                    attr = (pos_args, name_pair_args, annotate_args)
                 tokens.append(
                     Token(
                         type_ = opname,
-                        attr = (pos_args, name_pair_args, annotate_args),
+                        attr = attr,
                         pattr = pattr,
                         offset = inst.offset,
                         linestart = inst.starts_line,

uncompyle6/semantics/make_function.py

@@ -461,7 +461,17 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
             defparams = node[:args_node.attr[0]]
         pos_args, kw_args, annotate_argc  = args_node.attr
     else:
-        defparams = node[:args_node.attr]
+        if self.version < 3.6:
+            defparams = node[:args_node.attr]
+        else:
+            default, kw, annotate, closure = args_node.attr
+            # FIXME: start here.
+            defparams = []
+            # if default:
+            #     defparams = node[-(2 +  kw + annotate  + closure)]
+            # else:
+            #     defparams = []
+
         kw_args  = 0
         pass
 

uncompyle6/semantics/pysource.py

@@ -424,7 +424,17 @@ class SourceWalker(GenericASTTraversal, object):
                             if i < num_kwargs:
                                 self.write(',')
                         self.prune()
+                        return
                     self.n_kwargs_only_36 = n_kwargs_only_36
+
+                    def n_return_closure(node):
+                        # Nothing should be output here
+                        self.prune()
+                        return
+                    self.n_return_closure = n_return_closure
+                    pass # version > 3.6
+                pass # version > 3.4
+            pass # version > 3.0
         return
 
     f = property(lambda s: s.params['f'],
@@ -1288,11 +1298,18 @@ class SourceWalker(GenericASTTraversal, object):
 
         if self.version > 3.0:
             if node == 'classdefdeco2':
-                currentclass = node[1][2].pattr
+                if self.version >= 3.6:
+                    class_code = node[1][1].pattr
+                else:
+                    class_code = node[1][2].pattr
                 buildclass = node
             else:
-                currentclass = node[1][0].pattr
+                if self.version >= 3.6:
-                buildclass = node[0]
+                    class_code = node[0][1][0].pattr
+                    buildclass = node[0]
+                else:
+                    class_code = node[1][0].pattr
+                    buildclass = node[0]
 
             assert 'mkfunc' == buildclass[1]
             mkfunc = buildclass[1]
@@ -1300,16 +1317,16 @@ class SourceWalker(GenericASTTraversal, object):
                 if 3.0 <= self.version <= 3.2:
                     for n in mkfunc:
                         if hasattr(n, 'attr') and iscode(n.attr):
-                            subclass = n.attr
+                            subclass_code = n.attr
                             break
                         elif n == 'expr':
-                            subclass = n[0].attr
+                            subclass_code = n[0].attr
                         pass
                     pass
                 else:
                     for n in mkfunc:
                         if hasattr(n, 'attr') and iscode(n.attr):
-                            subclass = n.attr
+                            subclass_code = n.attr
                             break
                         pass
                     pass
@@ -1321,10 +1338,10 @@ class SourceWalker(GenericASTTraversal, object):
                     # Python 3.3 classes with closures work like this.
                     # Note have to test before 3.2 case because
                     # index -2 also has an attr.
-                    subclass = load_closure[-3].attr
+                    subclass_code = load_closure[-3].attr
                 elif hasattr(load_closure[-2], 'attr'):
                     # Python 3.2 works like this
-                    subclass = load_closure[-2].attr
+                    subclass_code = load_closure[-2].attr
                 else:
                     raise 'Internal Error n_classdef: cannot find class body'
                 if hasattr(buildclass[3], '__len__'):
@@ -1333,18 +1350,21 @@ class SourceWalker(GenericASTTraversal, object):
                     subclass_info = buildclass[2]
                 else:
                     raise 'Internal Error n_classdef: cannot superclass name'
+            elif self.version >= 3.6 and node == 'classdefdeco2':
+                subclass_info = node
+                subclass_code = buildclass[1][0].attr
             else:
-                subclass = buildclass[1][0].attr
+                subclass_code = buildclass[1][0].attr
                 subclass_info = node[0]
         else:
             buildclass = node if (node == 'classdefdeco2') else node[0]
             build_list = buildclass[1][0]
             if hasattr(buildclass[-3][0], 'attr'):
-                subclass = buildclass[-3][0].attr
+                subclass_code = buildclass[-3][0].attr
-                currentclass = buildclass[0].pattr
+                class_code = buildclass[0].pattr
             elif hasattr(node[0][0], 'pattr'):
-                subclass = buildclass[-3][1].attr
+                subclass_code = buildclass[-3][1].attr
-                currentclass = node[0][0].pattr
+                class_code = node[0][0].pattr
             else:
                 raise 'Internal Error n_classdef: cannot find class name'
 
@@ -1353,7 +1373,7 @@ class SourceWalker(GenericASTTraversal, object):
         else:
             self.write('\n\n')
 
-        self.currentclass = str(currentclass)
+        self.currentclass = str(class_code)
         self.write(self.indent, 'class ', self.currentclass)
 
         if self.version > 3.0:
@@ -1364,7 +1384,7 @@ class SourceWalker(GenericASTTraversal, object):
 
         # class body
         self.indentMore()
-        self.build_class(subclass)
+        self.build_class(subclass_code)
         self.indentLess()
 
         self.currentclass = cclass