Start to handle CALL_FUNCTION_EX more accurately

admin-tools/how-to-make-a-release.md

@@ -52,7 +52,7 @@
 
     $ git commit -m"Get ready for release $VERSION" .
 
-# Check against all versions
+# Check against older versions
 
     $ source admin-tools/check-older-versions.sh
 

uncompyle6/parsers/parse25.py

@@ -29,7 +29,6 @@ class Python25Parser(Python26Parser):
         withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                      POP_BLOCK LOAD_CONST COME_FROM with_cleanup
 
-        store ::= STORE_FAST
         store ::= STORE_NAME
 
         tryelsestmt    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK

uncompyle6/parsers/parse3.py

@@ -656,7 +656,11 @@ class Python3Parser(PythonParser):
                 rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
                         'expr32 ' * int((v//32) % 32) +
                         'expr ' * (v % 32) + opname)
-                self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
+                self.custom_classfunc_rule(opname, token, customize,
+                                           seen_LOAD_BUILD_CLASS,
+                                           seen_GET_AWAITABLE_YIELD_FROM, tokens[i+1])
+                continue
             elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
                 v = token.attr
 
@@ -693,14 +697,11 @@ class Python3Parser(PythonParser):
                         'expr ::= build_slice3',
                         'build_slice3 ::= expr expr expr BUILD_SLICE_3',
                         ], customize)
-            elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
+            elif (opname in frozenset(('CALL_FUNCTION',
-                v = token.attr
+                                       'CALL_FUNCTION_EX',
-                rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
+                                       'CALL_FUNCTION_EX_KW',
-                        'expr32 ' * int((v//32) % 32) +
+                                       'CALL_FUNCTION_VAR',
-                        'expr ' * (v % 32) + opname)
+                                       'CALL_FUNCTION_VAR_KW'))
-                self.add_unique_rule(rule, opname, token.attr, customize)
-            elif (opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
-                             'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_EX_KW')
                   or opname.startswith('CALL_FUNCTION_KW')):
                 self.custom_classfunc_rule(opname, token, customize,
                                            seen_LOAD_BUILD_CLASS,

uncompyle6/parsers/parse31.py

@@ -28,7 +28,6 @@ class Python31Parser(Python32Parser):
                            POP_BLOCK LOAD_CONST COME_FROM_FINALLY
                            load del_stmt WITH_CLEANUP END_FINALLY
 
-        store ::= STORE_FAST
         store ::= STORE_NAME
         load  ::= LOAD_FAST
         load  ::= LOAD_NAME

uncompyle6/parsers/parse36.py

@@ -4,7 +4,7 @@ spark grammar differences over Python 3.5 for Python 3.6.
 """
 from __future__ import print_function
 
-from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse35 import Python35Parser
 
@@ -27,16 +27,9 @@ class Python36Parser(Python35Parser):
         return_if_lambda   ::= RETURN_END_IF_LAMBDA
 
 
-        func_args36   ::= expr BUILD_TUPLE_0
-        call          ::= func_args36 unmapexpr CALL_FUNCTION_EX
-        call          ::= func_args36 build_map_unpack_with_call CALL_FUNCTION_EX_KW_1
-
         withstmt      ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
                           WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        call          ::= expr expr CALL_FUNCTION_EX
-        call          ::= expr expr expr CALL_FUNCTION_EX_KW_1
-
         for_block       ::= l_stmts_opt come_from_loops JUMP_BACK
         come_from_loops ::= COME_FROM_LOOP*
 
@@ -112,11 +105,38 @@ class Python36Parser(Python35Parser):
                               possible_class_decorator,
                               seen_GET_AWAITABLE_YIELD_FROM, next_token):
         if opname.startswith('CALL_FUNCTION_KW'):
+            self.addRule("expr ::= call_kw", nop_func)
             values = 'expr ' * token.attr
-            rule = 'call ::= expr kwargs_only_36 {token.kind}'.format(**locals())
+            rule = 'call_kw ::= expr kwargs_36 {token.kind}'.format(**locals())
-            self.add_unique_rule(rule, token.kind, token.attr, customize)
+            self.addRule(rule, nop_func)
-            rule = 'kwargs_only_36 ::= {values} LOAD_CONST'.format(**locals())
+            rule = 'kwargs_36 ::= {values} LOAD_CONST'.format(**locals())
             self.add_unique_rule(rule, token.kind, token.attr, customize)
+        elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
+            v = token.attr
+            rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
+                    'expr32 ' * int((v//32) % 32) +
+                    'expr ' * (v % 32) + opname)
+            self.addRule(rule, nop_func)
+        elif opname.startswith('CALL_FUNCTION_EX'):
+            self.addRule("expr ::= call_ex", nop_func)
+            if token.attr == 0:
+                self.addRule("""
+                             unpack_list ::= list
+                             call_ex     ::= expr unpack_list CALL_FUNCTION_EX
+                             """, nop_func)
+            else:
+                self.addRule(
+                    'call_ex ::= expr build_tuple_unpack_with_call CALL_FUNCTION_EX',
+                    nop_func)
+            pass
+        elif opname == 'CALL_FUNCTION_EX_KW':
+            args_pos, args_kw = self.get_pos_kw(token)
+            uniq_param = args_kw + args_pos
+            self.addRule("expr ::= call_ex_kw", nop_func)
+            rule = ('call_ex_kw ::= '
+                    'expr build_tuple_unpack_with_call build_map_unpack_with_call '
+                    'CALL_FUNCTION_EX_KW')
+            self.addRule(rule, nop_func)
         else:
             super(Python36Parser, self).custom_classfunc_rule(opname, token,
                                                               customize,

uncompyle6/scanners/scanner36.py

@@ -31,14 +31,14 @@ class Scanner36(Scanner3):
             # The lowest bit of flags indicates whether the
             # var-keyword argument is placed at the top of the stack
             if t.op == self.opc.CALL_FUNCTION_EX and t.attr & 1:
-                t.type = 'CALL_FUNCTION_EX_KW'
+                t.kind = 'CALL_FUNCTION_EX_KW'
                 pass
             elif t.op == self.opc.CALL_FUNCTION_KW:
-                t.type = 'CALL_FUNCTION_KW_{t.attr}'.format(**locals())
+                t.kind = 'CALL_FUNCTION_KW_{t.attr}'.format(**locals())
             elif t.op == self.opc.BUILD_TUPLE_UNPACK_WITH_CALL:
-                t.type = 'BUILD_TUPLE_UNPACK_WITH_CALL_%d' % t.attr
+                t.kind = 'BUILD_TUPLE_UNPACK_WITH_CALL_%d' % t.attr
             elif t.op == self.opc.BUILD_MAP_UNPACK_WITH_CALL:
-                t.type = 'BUILD_MAP_UNPACK_WITH_CALL_%d' % t.attr
+                t.kind = 'BUILD_MAP_UNPACK_WITH_CALL_%d' % t.attr
             pass
         return tokens, customize
 

uncompyle6/semantics/pysource.py

@@ -442,7 +442,60 @@ class SourceWalker(GenericASTTraversal, object):
                     if version >= 3.6:
                         TABLE_DIRECT.update({
                             'try_except36': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
-                            })
+                            'unpack_list':  ( '*%c', (0, 'list') ),
+                            'call_ex' : (
+                                '%c(%c)',
+                                (0, 'expr'), 1),
+                        })
+
+                        def n_build_tuple_unpack_with_call(node):
+                            if node[0] == 'expr':
+                                first = node[0][0]
+                            else:
+                                first = node[0]
+                            if first == 'tuple':
+                                flat_elems = []
+                                # Note: don't iteratee over last element which is a
+                                # BUILD_TUPLE...
+                                for elem in first[:-1]:
+                                    if elem == 'expr1024':
+                                        for subelem in elem:
+                                            for subsubelem in subelem:
+                                                flat_elems.append(subsubelem)
+                                    elif elem == 'expr32':
+                                        for subelem in elem:
+                                            flat_elems.append(subelem)
+                                    else:
+                                        flat_elems.append(elem)
+                                        pass
+                                    pass
+
+                                self.indent_more(INDENT_PER_LEVEL)
+                                sep = ''
+
+                                for elem in flat_elems:
+                                    if elem in ('ROT_THREE', 'EXTENDED_ARG'):
+                                        continue
+                                    assert elem == 'expr'
+                                    line_number = self.line_number
+                                    value = self.traverse(elem)
+                                    if line_number != self.line_number:
+                                        sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
+                                    self.write(sep, value)
+                                    sep = ', '
+
+                            self.indent_less(INDENT_PER_LEVEL)
+
+                            btuwc = node[-1]
+                            assert btuwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
+                            for n in node[1:-1]:
+                                self.f.write(', *')
+                                self.preorder(n)
+                                pass
+                            self.prune()
+                            return
+
+                        self.n_build_tuple_unpack_with_call = n_build_tuple_unpack_with_call
 
                     def n_async_call(node):
                         self.f.write('async ')