Merge branch 'master' into python-2.4

test/simple_source/bug27+/03_if_1_else.py

@@ -0,0 +1 @@
+1 if 1 else __file__

test/simple_source/bug31/04_def_annotate.py

@@ -17,3 +17,14 @@ def div(a: dict(type=float, help='the dividend'),
     ) -> dict(type=float, help='the result of dividing a by b'):
     """Divide a by b"""
     return a / b
+
+class TestSignatureObject(unittest.TestCase):
+    def test_signature_on_wkwonly(self):
+        def test(*, a:float, b:str) -> int:
+            pass
+
+class SupportsInt(_Protocol):
+
+    @abstractmethod
+    def __int__(self) -> int:
+        pass

test/simple_source/stmts/02_while1else.py

@@ -0,0 +1,8 @@
+# From Python-3.5.2/Lib/multiprocessing/connection.py
+
+def PipeClient(address):
+    while 1:
+        z = 2
+    else:
+        raise
+    return

uncompyle6/parser.py

@@ -512,8 +512,12 @@ class PythonParser(GenericASTBuilder):
         expr ::= conditional
         conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
         conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr
+
         expr ::= conditionalnot
-        conditionalnot ::= expr jmp_true expr _jump expr COME_FROM
+        conditionalnot  ::= expr jmp_true expr _jump expr COME_FROM
+
+        expr            ::= conditionalTrue
+        conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM
 
         ret_expr ::= expr
         ret_expr ::= ret_and

uncompyle6/parsers/parse2.py

@@ -42,7 +42,8 @@ class Python2Parser(PythonParser):
         while1stmt     ::= SETUP_LOOP l_stmts     JUMP_BACK COME_FROM
         while1stmt     ::= SETUP_LOOP l_stmts     JUMP_BACK POP_BLOCK COME_FROM
 
-        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK           else_suite COME_FROM
 
         exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
         exec_stmt ::= expr exprlist EXEC_STMT

uncompyle6/parsers/parse23.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016-2017 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 #  Copyright (c) 1999 John Aycock
 
@@ -14,6 +14,17 @@ class Python23Parser(Python24Parser):
 
     def p_misc23(self, args):
         '''
+        # Python 2.4 only adds something like the below for if 1:
+        # However we will just treat it as a noop (which of course messes up
+        # simple verify of bytecode.
+        # See also below in reduce_is_invalid where we check that the JUMP_FORWARD
+        # target matches the COME_FROM target
+        stmt     ::= if1_stmt
+        if1_stmt ::= JUMP_FORWARD JUMP_IF_FALSE THEN POP_TOP COME_FROM
+                     stmts
+                     JUMP_FORWARD COME_FROM POP_TOP COME_FROM
+
+
         # Used to keep semantic positions the same across later versions
         # of Python
         _while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM
@@ -33,6 +44,23 @@ class Python23Parser(Python24Parser):
         lc_body ::= LOAD_FAST expr LIST_APPEND
         '''
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python23Parser, self).add_custom_rules(tokens, customize)
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python24Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+
+        # FiXME: this code never gets called...
+        lhs = rule[0]
+        if lhs == 'nop_stmt':
+            return not int(tokens[first].pattr) == tokens[last].offset
+
+        return False
+
 class Python23ParserSingle(Python23Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse24.py

@@ -14,6 +14,15 @@ class Python24Parser(Python25Parser):
 
     def p_misc24(self, args):
         '''
+        # Python 2.4 only adds something like the below for if 1:
+        # However we will just treat it as a noop (which of course messes up
+        # simple verify of bytecode.
+        # See also below in reduce_is_invalid where we check that the JUMP_FORWARD
+        # target matches the COME_FROM target
+        stmt     ::= nop_stmt
+        nop_stmt ::= JUMP_FORWARD POP_TOP COME_FROM
+
+
         # 2.5+ has two LOAD_CONSTs, one for the number '.'s in a relative import
         # keep positions similar to simplify semantic actions
 
@@ -37,6 +46,25 @@ class Python24Parser(Python25Parser):
         gen_comp_body ::= expr YIELD_VALUE
         '''
 
+    def add_custom_rules(self, tokens, customize):
+        super(Python24Parser, self).add_custom_rules(tokens, customize)
+        if self.version == 2.4:
+            self.check_reduce['nop_stmt'] = 'tokens'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python24Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+
+        # FiXME: this code never gets called...
+        lhs = rule[0]
+        if lhs == 'nop_stmt':
+            return not int(tokens[first].pattr) == tokens[last].offset
+
+        return False
+
 class Python24ParserSingle(Python24Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse3.py

@@ -154,7 +154,7 @@ class Python3Parser(PythonParser):
         # of missing "else" clauses. Therefore we include grammar
         # rules with and without ELSE.
 
-        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
+        ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except
         ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
@@ -273,12 +273,14 @@ class Python3Parser(PythonParser):
         stmt ::= funcdef_annotate
         funcdef_annotate ::= mkfunc_annotate designator
 
+        mkfuncdeco0 ::= mkfunc_annotate
+
         # This has the annotation value.
         # LOAD_NAME is used in an annotation type like
         # int, float, str
         annotate_arg    ::= LOAD_NAME
         # LOAD_CONST is used in an annotation string
-        annotate_arg    ::= LOAD_CONST
+        annotate_arg    ::= expr
 
         # This stores the tuple of parameter names
         # that have been annotated
@@ -379,6 +381,8 @@ class Python3Parser(PythonParser):
 
         while1stmt        ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
 
+        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK
+                              else_suite COME_FROM_LOOP
 
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
@@ -403,7 +407,6 @@ class Python3Parser(PythonParser):
         conditional    ::= expr jmp_false expr jf_else expr COME_FROM
         conditionalnot ::= expr jmp_true  expr jf_else expr COME_FROM
 
-
         expr ::= LOAD_CLASSNAME
 
         # Python 3.4+
@@ -708,9 +711,6 @@ class Python3Parser(PythonParser):
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
                 if opname.startswith('MAKE_FUNCTION_A'):
-                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                    self.add_unique_rule(rule, opname, token.attr, customize)
                     if self.version >= 3.3:
                         rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
                                 (('pos_arg ' * (args_pos)),

uncompyle6/parsers/parse35.py

@@ -19,8 +19,11 @@ class Python35Parser(Python34Parser):
         # I'm sure by the time Python 4 comes around these will be turned
         # into special opcodes
 
-        while1stmt  ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
-                        POP_BLOCK COME_FROM_LOOP
+        while1stmt     ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
+                           POP_BLOCK COME_FROM_LOOP
+        while1stmt     ::= SETUP_LOOP l_stmts POP_BLOCK COME_FROM_LOOP
+        while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK
+                           POP_BLOCK else_suite COME_FROM_LOOP
 
         # Python 3.5+ Await statement
         stmt ::= await_stmt

uncompyle6/scanners/scanner27.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, 2016 by Rocky Bernstein
+# Copyright (c) 2015-2017 by Rocky Bernstein
 """
 Python 2.7 bytecode ingester.
 
@@ -46,7 +46,7 @@ class Scanner27(Scanner2):
             self.opc.DELETE_SLICE_2,   self.opc.DELETE_SLICE_3,
         ])
 
-        # opcodes with expect a variable number pushed values whose
+        # opcodes which expect a variable number pushed values and whose
         # count is in the opcode. For parsing we generally change the
         # opcode name to include that number.
         varargs_ops = set([

uncompyle6/semantics/check_ast.py

@@ -10,7 +10,7 @@ before reduction and don't reduce when there is a problem.
 
 def checker(ast, in_loop, errors):
     in_loop = in_loop or ast.type in ('while1stmt', 'whileTruestmt',
-                                      'whilestmt', 'whileelsestmt',
+                                      'whilestmt', 'whileelsestmt', 'while1elsestmt',
                                       'for_block')
     if ast.type in ('augassign1', 'augassign2') and ast[0][0] == 'and':
         text = str(ast)

uncompyle6/semantics/consts.py

@@ -170,6 +170,7 @@ TABLE_DIRECT = {
     'or':           	( '%c or %c', 0, 2 ),
     'ret_or':           	( '%c or %c', 0, 2 ),
     'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
+    'conditionalTrue':  ( '%p if 1 else %p', (0, 27), (2, 27)),
     'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
     'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
     'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),

uncompyle6/semantics/fragments.py

@@ -77,7 +77,9 @@ from uncompyle6.semantics.consts import (
     TABLE_DIRECT, escape, minint, MAP
     )
 
-from uncompyle6.semantics.make_function import find_all_globals, find_none
+from uncompyle6.semantics.make_function import (
+    find_all_globals, find_none, code_has_star_arg, code_has_star_star_arg
+)
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
@@ -1668,10 +1670,10 @@ class FragmentsWalker(pysource.SourceWalker, object):
               name, default in map(lambda *tup:tup, *tup)]
         params.reverse() # back to correct order
 
-        if 4 & code.co_flags:	# flag 2 -> variable number of args
+        if code_has_star_arg(code):
             params.append('*%s' % code.co_varnames[argc])
             argc += 1
-        if 8 & code.co_flags:	# flag 3 -> keyword args
+        if code_has_star_star_arg(code):
             params.append('**%s' % code.co_varnames[argc])
             argc += 1
 

uncompyle6/semantics/make_function.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015, 2016 by Rocky Bernstein
+#  Copyright (c) 2015-2017 by Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 """
 All the crazy things we have to do to handle Python functions
@@ -40,6 +40,17 @@ def find_none(node):
             return True
     return False
 
+# FIXME: put this in xdis
+def code_has_star_arg(code):
+    """Return True iff
+    the code object has a variable positional parameter (*args-like)"""
+    return (code.co_flags & 4) != 0
+
+def code_has_star_star_arg(code):
+    """Return True iff
+    The code object has a variable keyword parameter (**kwargs-like)."""
+    return (code.co_flags & 8) != 0
+
 # FIXME: DRY the below code...
 
 def make_function3_annotate(self, node, isLambda, nested=1,
@@ -81,8 +92,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         j = annotate_last-1
         l = -len(node)
         while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'):
-            annotate_args[annotate_tup[i]] = (node[j][0].attr,
-                                              node[j][0] == 'LOAD_CONST')
+            annotate_args[annotate_tup[i]] = node[j][0]
             i -= 1
             j -= 1
 
@@ -91,9 +101,12 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         # positional args are before kwargs
         defparams = node[:args_node.attr[0]]
         pos_args, kw_args, annotate_argc  = args_node.attr
+        if 'return' in annotate_args.keys():
+            annotate_argc = len(annotate_args) - 1
     else:
         defparams = node[:args_node.attr]
         kw_args  = 0
+        annotate_argc = 0
         pass
 
     if 3.0 <= self.version <= 3.2:
@@ -139,7 +152,7 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     indent = ' ' * l
     line_number = self.line_number
 
-    if 4 & code.co_flags:	# flag 2 -> variable number of args
+    if code_has_star_arg(code):
         self.write('*%s' % code.co_varnames[argc + kw_pairs])
         argc += 1
 
@@ -184,10 +197,12 @@ def make_function3_annotate(self, node, isLambda, nested=1,
             else:
                 suffix = ', '
 
+
     # self.println(indent, '#flags:\t', int(code.co_flags))
-    if kw_args > 0:
-        if not (4 & code.co_flags):
+    if kw_args + annotate_argc > 0:
+        if not code_has_star_arg(code):
             if argc > 0:
+
                 self.write(", *, ")
             else:
                 self.write("*, ")
@@ -210,9 +225,29 @@ def make_function3_annotate(self, node, isLambda, nested=1,
                 i += 1
                 pass
             pass
+        annotate_args = []
+        for n in node:
+            if n == 'annotate_arg':
+                annotate_args.append(n[0])
+            elif n == 'annotate_tuple':
+                t = n[0].attr
+                if t[-1] == 'return':
+                    t = t[0:-1]
+                    annotate_args = annotate_args[:-1]
+                    pass
+                last = len(annotate_args) - 1
+                for i in range(len(annotate_args)):
+                    self.write("%s: " % (t[i]))
+                    self.preorder(annotate_args[i])
+                    if i < last:
+                        self.write(', ')
+                        pass
+                    pass
+                break
+            pass
         pass
 
-    if 8 & code.co_flags:	# flag 3 -> keyword args
+    if code_has_star_star_arg(code):
         if argc > 0:
             self.write(', ')
         self.write('**%s' % code.co_varnames[argc + kw_pairs])
@@ -295,6 +330,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
     else:
         defparams = node[:args_node.attr]
         kw_args  = 0
+        annotate_argc  = 0
         pass
 
     lambda_index = None
@@ -337,7 +373,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
               name, default in map(lambda *tup:tup, *tup)]
     params.reverse() # back to correct order
 
-    if 4 & code.co_flags:	# flag 2 -> variable number of args
+    if code_has_star_arg(code):
         params.append('*%s' % code.co_varnames[argc])
         argc += 1
 
@@ -365,7 +401,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
             break
         pass
 
-    if 8 & code.co_flags:	# flag 3 -> keyword args
+    if code_has_star_star_arg(code):
         if argc > 0:
             self.write(', ')
         self.write('**%s' % code.co_varnames[argc + kw_pairs])
@@ -478,7 +514,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
               name, default in map(lambda *tup:tup, *tup)]
         params.reverse() # back to correct order
 
-        if 4 & code.co_flags:	# flag 2 -> variable number of args
+        if code_has_star_arg(code):
             if self.version > 3.0:
                 params.append('*%s' % code.co_varnames[argc + kw_pairs])
             else:
@@ -504,7 +540,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
         indent = ' ' * l
         line_number = self.line_number
 
-        if 4 & code.co_flags:	# flag 2 -> variable number of args
+        if code_has_star_arg(code):
             self.write('*%s' % code.co_varnames[argc + kw_pairs])
             argc += 1
 
@@ -561,7 +597,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
             pass
         pass
 
-    if 8 & code.co_flags:	# flag 3 -> keyword args
+    if code_has_star_star_arg(code):
         if argc > 0:
             self.write(', ')
         self.write('**%s' % code.co_varnames[argc + kw_pairs])

uncompyle6/semantics/pysource.py

@@ -216,6 +216,11 @@ class SourceWalker(GenericASTTraversal, object):
                 'importlist2':	( '%C', (0, maxint, ', ') ),
                 })
             if version <= 2.4:
+                if version == 2.3:
+                    TABLE_DIRECT.update({
+                        'if1_stmt':	( '%|if 1\n%+%c%-', 5 )
+                    })
+
                 global NAME_MODULE
                 NAME_MODULE = AST('stmt',
                                   [ AST('assign',
@@ -1805,7 +1810,8 @@ class SourceWalker(GenericASTTraversal, object):
                     str += '*%c, **%c)'
                     # Python 3.5 only puts optional args (the VAR part)
                     # lowest down the stack
-                    if self.version == 3.5:
+                    na = (v & 0xff)  # positional parameters
+                    if self.version == 3.5 and na == 0:
                         if p2[2]: p2 = (2, -2, ', ')
                         entry = (str, 0, p2, 1, -2)
                     else: