Merge branch 'master' into python-2.4

README.rst

@@ -138,10 +138,13 @@ on the lower end Python versions which is more difficult for us to
 handle since we don't have a Python interpreter for versions 1.5, 1.6,
 and 2.0.
 
-Python 3.0 support is weak; Python 3.5 largely works, but still has
-some bugs in it.  Python 3.6 changes things drastically by using word
-codes rather than byte codes. That has been addressed, but then it also
-changes function call opcodes and its semantics.
+In the Python 3 series, Python support is is strongest around 3.4 or
+3.3 and drops off as you move further away from those versions. Python
+3.5 largely works, but still has some bugs in it and is missing some
+opcodes.  Python 3.6 changes things drastically by using word codes
+rather than byte codes. That has been addressed, but then it also
+changes function call opcodes and its semantics and has more problems
+with control flow than 3.5 has.
 
 Currently not all Python magic numbers are supported. Specifically in
 some versions of Python, notably Python 3.6, the magic number has

test/simple_source/bug31/04_def_annotate.py

@@ -8,3 +8,12 @@ def open(file, mode = "r", buffering = None,
          encoding = None, errors = None,
          newline = None, closefd = True) -> "IOBase":
     return text
+
+def foo(x: 'an argument that defaults to 5' = 5):
+    print(x)
+
+def div(a: dict(type=float, help='the dividend'),
+        b: dict(type=float, help='the divisor (must be different than 0)')
+    ) -> dict(type=float, help='the result of dividing a by b'):
+    """Divide a by b"""
+    return a / b

test/simple_source/bug35/03_async_await.py

@@ -0,0 +1,26 @@
+# Python 3.5+ async and await
+async def await_test(asyncio):
+    reader, writer = await asyncio.open_connection(80)
+    await bar()
+
+async def afor_test():
+
+    async for i in [1,2,3]:
+        x = i
+
+
+async def afor_else_test():
+
+    async for i in [1,2,3]:
+        x = i
+    else:
+        z = 4
+
+
+async def awith_test():
+    async with i:
+        print(i)
+
+async def awith_as_test():
+    async with 1 as i:
+        print(i)

test/simple_source/bug35/03_double_star_unpack.py

@@ -0,0 +1,9 @@
+# Bug in Python 3.5 is getting the two star'd arguments right.
+def sum(a,b,c,d):
+    return a + b + c + d
+
+args=(1,2)
+sum(*args, *args)
+
+# FIXME: this is handled incorrectly
+# (*c,) = (3,4)

test/simple_source/bug35/03_while-if-break.py

@@ -0,0 +1,7 @@
+# Python 3.5 and 3.6 break inside a
+# while True and if / break
+def display_date(loop):
+    while True:
+        if loop.time():
+            break
+        x = 5

test/test_pyenvlib.py

@@ -31,7 +31,7 @@ TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
                '2.7.10', '2.7.11', '2.7.12',
                '3.0.1', '3.1.5', '3.2.6',
                '3.3.5', '3.3.6',
-               '3.4.2', '3.5.1')
+               '3.4.2', '3.5.1', '3.6.0')
 
 target_base = '/tmp/py-dis/'
 lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')

uncompyle6/__init__.py

@@ -47,7 +47,10 @@ import uncompyle6.semantics.pysource
 import uncompyle6.semantics.fragments
 
 # Export some functions
-from uncompyle6.main import uncompyle_file
+from uncompyle6.main import decompile_file
+
+# For compaitility
+uncompyle_file = decompile_file
 
 # Conventience functions so you can say:
 # from uncompyle6 import deparse_code

uncompyle6/main.py

@@ -10,7 +10,7 @@ from uncompyle6.linenumbers import line_number_mapping
 
 from xdis.load import load_module
 
-def uncompyle(
+def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
         timestamp=None, showgrammar=False, code_objects={},
         source_size=None, is_pypy=False, magic_int=None):
@@ -52,8 +52,10 @@ def uncompyle(
                           showgrammar, code_objects=code_objects,
                           is_pypy=is_pypy)
 
+# For compatiblity
+uncompyle = decompile
 
-def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
+def decompile_file(filename, outstream=None, showasm=None, showast=False,
                    showgrammar=False):
     """
     decompile Python byte-code file (.pyc)
@@ -66,16 +68,20 @@ def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
 
     if type(co) == list:
         for con in co:
-            uncompyle(version, con, outstream, showasm, showast,
+            decompile(version, con, outstream, showasm, showast,
                       timestamp, showgrammar, code_objects=code_objects,
                       is_pypy=is_pypy, magic_int=magic_int)
     else:
-        uncompyle(version, co, outstream, showasm, showast,
+        decompile(version, co, outstream, showasm, showast,
                   timestamp, showgrammar,
                   code_objects=code_objects, source_size=source_size,
                   is_pypy=is_pypy, magic_int=magic_int)
     co = None
 
+# For compatiblity
+uncompyle_file = decompile_file
+
+
 # FIXME: combine into an options parameter
 def main(in_base, out_base, files, codes, outfile=None,
          showasm=None, showast=False, do_verify=False,
@@ -105,12 +111,6 @@ def main(in_base, out_base, files, codes, outfile=None,
 
     tot_files = okay_files = failed_files = verify_failed_files = 0
 
-    # for code in codes:
-    #    version = sys.version[:3] # "2.5"
-    #    with open(code, "r") as f:
-    #        co = compile(f.read(), "", "exec")
-    #    uncompyle(sys.version[:3], co, sys.stdout, showasm=showasm, showast=showast)
-
     for filename in files:
         infile = os.path.join(in_base, filename)
         if not os.path.exists(infile):
@@ -149,7 +149,7 @@ def main(in_base, out_base, files, codes, outfile=None,
 
         # Try to uncompile the input file
         try:
-            uncompyle_file(infile, outstream, showasm, showast, showgrammar)
+            decompile_file(infile, outstream, showasm, showast, showgrammar)
             tot_files += 1
         except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
             sys.stdout.write("\n")

uncompyle6/parsers/parse3.py

@@ -451,6 +451,15 @@ class Python3Parser(PythonParser):
                 ('kwarg ' * args_kw) +
                 'expr ' * nak + token.type)
         self.add_unique_rule(rule, token.type, args_pos, customize)
+        if self.version >= 3.5:
+            rule = ('async_call_function ::= expr ' +
+                    ('pos_arg ' * args_pos) +
+                    ('kwarg ' * args_kw) +
+                    'expr ' * nak + token.type +
+                    ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
+            self.add_unique_rule(rule, token.type, args_pos, customize)
+            self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
+
         rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                 %  (('expr ' * (args_pos-1)), opname, args_pos))
         self.add_unique_rule(rule, token.type, args_pos, customize)
@@ -477,8 +486,15 @@ class Python3Parser(PythonParser):
 
             # build_class (see load_build_class)
 
+            # Even the below say _list, in the semantic rules we
+            # disambiguate tuples, and sets from lists
+
             build_list ::= {expr}^n BUILD_LIST_n
             build_list ::= {expr}^n BUILD_TUPLE_n
+            build_list ::= {expr}^n BUILD_SET_n
+            build_list ::= {expr}^n BUILD_LIST_UNPACK_n
+            build_list ::= {expr}^n BUILD_SET_UNPACK_n
+            build_list ::= {expr}^n BUILD_TUPLE_UNPACK_n
 
             load_closure  ::= {LOAD_CLOSURE}^n BUILD_TUPLE_n
             # call_function (see custom_classfunc_rule)
@@ -637,6 +653,27 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= kwargs %sexpr %s' %
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                if opname.startswith('MAKE_FUNCTION_A'):
+                    # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
+                    #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    if self.version >= 3.3:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                    else:
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('annotate_arg ' * (annotate_args-1)), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
+                                (('pos_arg ' * (args_pos)),
+                                 ('call_function ' * (annotate_args-1)), opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base == 'CALL_METHOD':
                 # PyPy only - DRY with parse2
 
@@ -697,6 +734,8 @@ class Python3Parser(PythonParser):
         self.check_reduce['augassign1'] = 'AST'
         self.check_reduce['augassign2'] = 'AST'
         self.check_reduce['while1stmt'] = 'noAST'
+        self.check_reduce['annotate_tuple'] = 'noAST'
+        self.check_reduce['kwarg'] = 'noAST'
         # FIXME: remove parser errors caused by the below
         # self.check_reduce['while1elsestmt'] = 'noAST'
         return
@@ -705,6 +744,10 @@ class Python3Parser(PythonParser):
         lhs = rule[0]
         if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
             return True
+        elif lhs == 'annotate_tuple':
+            return not isinstance(tokens[first].attr, tuple)
+        elif lhs == 'kwarg':
+            return not isinstance(tokens[first].attr, str)
         elif lhs == 'while1elsestmt':
             # if SETUP_LOOP target spans the else part, then this is
             # not while1else. Also do for whileTrue?

uncompyle6/parsers/parse30.py

@@ -14,6 +14,11 @@ class Python30Parser(Python3Parser):
         stmt         ::= store_locals
         store_locals ::= LOAD_FAST STORE_LOCALS
 
+        # FIXME: combine with parse3.2
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
+                              COME_FROM_LOOP
+        whileTruestmt     ::= SETUP_LOOP return_stmts
+                              COME_FROM_LOOP
 
         # In many ways Python 3.0 code generation is more like Python 2.6 than
         # it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below

uncompyle6/parsers/parse31.py

@@ -35,16 +35,8 @@ class Python31Parser(Python32Parser):
 
     def add_custom_rules(self, tokens, customize):
         super(Python31Parser, self).add_custom_rules(tokens, customize)
-        for i, token in enumerate(tokens):
-            opname = token.type
-            if opname.startswith('MAKE_FUNCTION_A'):
-                args_pos, args_kw, annotate_args  = token.attr
-                # Check that there are 2 annotated params?
-                # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
-                #         ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
-                rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                        (('pos_arg ' * (args_pos)),
-                         ('annotate_arg ' * (annotate_args-1)), opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+        return
+    pass
+
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse32.py

@@ -44,6 +44,9 @@ class Python32Parser(Python3Parser):
                         (('pos_arg ' * (args_pos)),
                          ('annotate_arg ' * (annotate_args-1)), opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+            return
+        pass
 
 
 class Python32ParserSingle(Python32Parser, PythonParserSingle):

uncompyle6/parsers/parse35.py

@@ -15,20 +15,80 @@ class Python35Parser(Python34Parser):
 
     def p_35on(self, args):
         """
+        # The number of canned instructions in new statements is mind boggling.
+        # I'm sure by the time Python 4 comes around these will be turned
+        # into special opcodes
+
+        # Python 3.5+ Await statement
+        stmt ::= await_stmt
+        await_stmt ::= call_function GET_AWAITABLE LOAD_CONST YIELD_FROM POP_TOP
+
         # Python 3.5+ has WITH_CLEANUP_START/FINISH
 
-        withstmt ::= expr SETUP_WITH exprlist suite_stmts_opt
+        withstmt    ::= expr
+                        SETUP_WITH exprlist suite_stmts_opt
                         POP_BLOCK LOAD_CONST COME_FROM_WITH
                         WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
+        withstmt   ::= expr
+                       SETUP_WITH POP_TOP suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM_WITH
                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
+        withasstmt ::= expr
+                       SETUP_WITH designator suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM_WITH
                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
+
+        # Python 3.5+ async additions
+
+        stmt            ::= async_with_stmt
+        async_with_stmt ::= expr
+                            BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
+                            POP_BLOCK LOAD_CONST
+                            WITH_CLEANUP_START
+                            GET_AWAITABLE LOAD_CONST YIELD_FROM
+                            WITH_CLEANUP_FINISH END_FINALLY
+
+        stmt               ::= async_with_as_stmt
+        async_with_as_stmt ::= expr
+                               BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               SETUP_ASYNC_WITH designator suite_stmts_opt
+                               POP_BLOCK LOAD_CONST
+                               WITH_CLEANUP_START
+                               GET_AWAITABLE LOAD_CONST YIELD_FROM
+                               WITH_CLEANUP_FINISH END_FINALLY
+
+
+        stmt               ::= async_for_stmt
+        async_for_stmt     ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               opt_come_from_loop
+
+        stmt               ::= async_forelse_stmt
+        async_forelse_stmt ::= SETUP_LOOP expr
+                               GET_AITER
+                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
+                               YIELD_FROM
+                               designator
+                               POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
+                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
+                               JUMP_ABSOLUTE END_FINALLY COME_FROM
+                               for_block POP_BLOCK JUMP_ABSOLUTE
+                               else_suite COME_FROM_LOOP
+
+
         inplace_op ::= INPLACE_MATRIX_MULTIPLY
         binary_op  ::= BINARY_MATRIX_MULTIPLY
 
@@ -38,13 +98,18 @@ class Python35Parser(Python34Parser):
         return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
 
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jf_else else_suitec
+
         # ifstmt ::= testexpr c_stmts_opt
 
+        iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
 
         # Python 3.3+ also has yield from. 3.5 does it
         # differently than 3.3, 3.4
 
         yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
+
+        _ifstmts_jump ::= c_stmts_opt COME_FROM
         """
 
     def add_custom_rules(self, tokens, customize):

uncompyle6/parsers/parse36.py

@@ -20,6 +20,9 @@ class Python36Parser(Python35Parser):
 
         func_args36   ::= expr BUILD_TUPLE_0
         call_function ::= func_args36 unmapexpr CALL_FUNCTION_EX
+
+        withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
+                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
         """
 
     def add_custom_rules(self, tokens, customize):

uncompyle6/scanner.py

@@ -224,7 +224,7 @@ class Scanner(object):
         for given opcode <op>.
         """
         if op < self.opc.HAVE_ARGUMENT:
-            return 1
+            return 2 if self.version >= 3.6 else 1
         else:
             if self.version >= 3.6:
                 return 2

uncompyle6/scanners/scanner2.py

@@ -490,20 +490,21 @@ class Scanner2(scan.Scanner):
         in python2.3+
         """
 
-        # TODO: check the struct boundaries more precisely -Dan
-
         code = self.code
 
         # Detect parent structure
         parent = self.structs[0]
         start  = parent['start']
         end    = parent['end']
+
+        # Pick inner-most parent for our offset
         for struct in self.structs:
-            _start = struct['start']
-            _end   = struct['end']
-            if (_start <= offset < _end) and (_start >= start and _end <= end):
-                start  = _start
-                end    = _end
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start  = current_start
+                end    = current_end
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:

uncompyle6/scanners/scanner3.py

@@ -328,13 +328,15 @@ class Scanner3(Scanner):
                 if target <= inst.offset:
                     next_opname = self.opname[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
+                        (self.version != 3.0 or (hasattr(inst, 'linestart'))) and
                         (next_opname not in ('END_FINALLY', 'POP_BLOCK',
                                             # Python 3.0 only uses POP_TOP
-                                            'POP_TOP')
-                        and inst.offset not in self.not_continue) or
-                        (tokens[-1].type == 'RETURN_VALUE' and
-                         self.version < 3.5)):
+                                            'POP_TOP'))):
+                        if (self.version >= 3.5 or
+                            (inst.offset not in self.not_continue) or
+                            (tokens[-1].type == 'RETURN_VALUE')):
                             opname = 'CONTINUE'
+                            pass
                     else:
                         opname = 'JUMP_BACK'
                         # FIXME: this is a hack to catch stuff like:
@@ -466,11 +468,15 @@ class Scanner3(Scanner):
             has_arg = (op >= op3.HAVE_ARGUMENT)
             if has_arg:
                 label = self.fixed_jumps.get(offset)
+                if self.version >= 3.6:
+                    oparg = code[offset+1]
+                else:
                     oparg = code[offset+1] + code[offset+2] * 256
+                next_offset = offset + self.op_size(op)
 
                 if label is None:
                     if op in op3.hasjrel and op != self.opc.FOR_ITER:
-                        label = offset + 3 + oparg
+                        label = next_offset + oparg
                     elif op in op3.hasjabs:
                         if op in self.jump_if_pop:
                             if oparg > offset:
@@ -500,7 +506,7 @@ class Scanner3(Scanner):
         prelim = self.all_instr(start, end, self.statement_opcodes)
 
         # Initialize final container with statements with
-        # preliminnary data
+        # preliminary data
         stmts = self.stmts = set(prelim)
 
         # Same for opcode sequences
@@ -579,11 +585,11 @@ class Scanner3(Scanner):
         op = self.code[offset]
         if self.version  >= 3.6:
             target = self.code[offset+1]
-            if op in op3.hasjrel:
+            if op in self.opc.hasjrel:
                 target += offset + 2
         else:
             target = self.code[offset+1] + self.code[offset+2] * 256
-            if op in op3.hasjrel:
+            if op in self.opc.hasjrel:
                 target += offset + 3
 
         return target
@@ -620,7 +626,10 @@ class Scanner3(Scanner):
             # Try to find the jump_back instruction of the loop.
             # It could be a return instruction.
 
+            if self.version <= 3.5:
                 start = offset+3
+            else:
+                start = offset+2
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
             self.setup_loop_targets[offset] = target
@@ -646,12 +655,11 @@ class Scanner3(Scanner):
                      and code[return_val_offset1] == self.opc.RETURN_VALUE)):
                     jump_back = None
             if not jump_back:
-                # loop suite ends in return
-                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE) + 1
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
                 if not jump_back:
                     return
 
-                jump_back += 1
+                jump_back += 2
                 if_offset = None
                 if code[self.prev_op[next_line_byte]] not in POP_JUMP_TF:
                     if_offset = self.prev[next_line_byte]
@@ -850,7 +858,8 @@ class Scanner3(Scanner):
                                      'start': start,
                                      'end': pre_rtarget})
                 self.not_continue.add(pre_rtarget)
-            elif code[pre_rtarget] == self.opc.RETURN_VALUE:
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -880,7 +889,12 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
                     self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+
 
         elif op == self.opc.SETUP_EXCEPT:
             target = self.get_target(offset)

uncompyle6/scanners/scanner30.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016, 2017 by Rocky Bernstein
 """
 Python 3.0 bytecode scanner/deparser
 
@@ -10,6 +10,8 @@ scanner routine for Python 3.
 from xdis.opcodes import opcode_30 as opc
 JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
 
+JUMP_TF = frozenset([opc.JUMP_IF_FALSE, opc.JUMP_IF_TRUE])
+
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner30(Scanner3):
 
@@ -18,6 +20,373 @@ class Scanner30(Scanner3):
         return
     pass
 
+    def detect_control_flow(self, offset, targets):
+        """
+        Detect structures and their boundaries to fix optimized jumps
+        Python 3.0 is more like Python 2.6 than it is Python 3.x.
+        So we have a special routine here.
+        """
+
+        code = self.code
+        op = code[offset]
+
+        # Detect parent structure
+        parent = self.structs[0]
+        start = parent['start']
+        end = parent['end']
+
+        # Pick inner-most parent for our offset
+        for struct in self.structs:
+            current_start = struct['start']
+            current_end   = struct['end']
+            if ((current_start <= offset < current_end)
+                and (current_start >= start and current_end <= end)):
+                start = current_start
+                end = current_end
+                parent = struct
+
+        if op == self.opc.SETUP_LOOP:
+            # We categorize loop types: 'for', 'while', 'while 1' with
+            # possibly suffixes '-loop' and '-else'
+            # Try to find the jump_back instruction of the loop.
+            # It could be a return instruction.
+
+            start = offset+3
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.setup_loop_targets[offset] = target
+            self.setup_loops[target] = offset
+
+            if target != end:
+                self.fixed_jumps[offset] = end
+
+            (line_no, next_line_byte) = self.lines[offset]
+            jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
+                                            next_line_byte, False)
+
+            if jump_back:
+                jump_forward_offset = jump_back+3
+            else:
+                jump_forward_offset = None
+
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
+                    jump_back = None
+            if not jump_back:
+                # loop suite ends in return
+                jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE)
+                if not jump_back:
+                    return
+
+                jump_back += 2
+                if_offset = None
+                if code[self.prev_op[next_line_byte]] not in JUMP_TF:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
+                    loop_type = 'while'
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
+                target = next_line_byte
+                end = jump_back + 3
+            else:
+                if self.get_target(jump_back) >= next_line_byte:
+                    jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE, start, False)
+                if end > jump_back+4 and self.is_jump_forward(end):
+                    if self.is_jump_forward(jump_back+4):
+                        if self.get_target(jump_back+4) == self.get_target(end):
+                            self.fixed_jumps[offset] = jump_back+4
+                            end = jump_back+4
+                elif target < offset:
+                    self.fixed_jumps[offset] = jump_back+4
+                    end = jump_back+4
+
+                target = self.get_target(jump_back)
+
+                if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
+                    loop_type = 'for'
+                else:
+                    loop_type = 'while'
+                    test = self.prev_op[next_line_byte]
+
+                    if test == offset:
+                        loop_type = 'while 1'
+                    elif self.code[test] in opc.hasjabs+opc.hasjrel:
+                        self.ignore_if.add(test)
+                        test_target = self.get_target(test)
+                        if test_target > (jump_back+3):
+                            jump_back = test_target
+                self.not_continue.add(jump_back)
+            self.loops.append(target)
+            self.structs.append({'type': loop_type + '-loop',
+                                   'start': target,
+                                   'end':   jump_back})
+            if jump_back+3 != end:
+                self.structs.append({'type': loop_type + '-else',
+                                       'start': jump_back+3,
+                                       'end':   end})
+        elif op in JUMP_TF:
+            start = offset + self.op_size(op)
+            target = self.get_target(offset)
+            rtarget = self.restrict_to_parent(target, parent)
+            prev_op = self.prev_op
+
+            # Do not let jump to go out of parent struct bounds
+            if target != rtarget and parent['type'] == 'and/or':
+                self.fixed_jumps[offset] = rtarget
+                return
+
+            # Does this jump to right after another conditional jump that is
+            # not myself?  If so, it's part of a larger conditional.
+            # rocky: if we have a conditional jump to the next instruction, then
+            # possibly I am "skipping over" a "pass" or null statement.
+
+            if ((code[prev_op[target]] in self.pop_jump_if_pop) and
+                (target > offset) and prev_op[target] != offset):
+                self.fixed_jumps[offset] = prev_op[target]
+                self.structs.append({'type': 'and/or',
+                                     'start': start,
+                                     'end': prev_op[target]})
+                return
+
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
+            if op == opc.JUMP_IF_FALSE:
+
+                # Search for another JUMP_IF_FALSE targetting the same op,
+                # in current statement, starting from current offset, and filter
+                # everything inside inner 'or' jumps and midline ifs
+                match = self.rem_or(start, self.next_stmt[offset],
+                                    opc.JUMP_IF_FALSE, target)
+
+                # If we still have any offsets in set, start working on it
+                if match:
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
+                            and self.remove_mid_line_ifs([offset]) and
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], JUMP_TF, target)))):
+                            pass
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
+                              and self.remove_mid_line_ifs([offset]) and
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                                 JUMP_TF, target))) |
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
+                                                                             (opc.JUMP_IF_FALSE,
+                                                                              opc.JUMP_IF_TRUE,
+                                                                              opc.JUMP_ABSOLUTE),
+                                                                             pre_rtarget, True)))))):
+                            pass
+                        else:
+                            fix = None
+                            jump_ifs = self.all_instr(start, self.next_stmt[offset],
+                                                      opc.JUMP_IF_FALSE)
+                            last_jump_good = True
+                            for j in jump_ifs:
+                                if target == self.get_target(j):
+                                    if self.lines[j].next == j + 3 and last_jump_good:
+                                        fix = j
+                                        break
+                                else:
+                                    last_jump_good = False
+                            self.fixed_jumps[offset] = fix or match[-1]
+                            return
+                    else:
+                        self.fixed_jumps[offset] = match[-1]
+                        return
+            # op == JUMP_IF_TRUE
+            else:
+                next = self.next_stmt[offset]
+                if prev_op[next] == offset:
+                    pass
+                elif self.is_jump_forward(next) and target == self.get_target(next):
+                    if code[prev_op[next]] == opc.JUMP_IF_FALSE:
+                        if (code[next] == self.opc.JUMP_FORWARD
+                            or target != rtarget
+                            or code[prev_op[pre_rtarget]] not in
+                            (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
+                            self.fixed_jumps[offset] = prev_op[next]
+                            return
+                elif (code[next] == self.opc.JUMP_ABSOLUTE and self.is_jump_forward(target) and
+                      self.get_target(target) == self.get_target(next)):
+                    self.fixed_jumps[offset] = prev_op[next]
+                    return
+
+            # Don't add a struct for a while test, it's already taken care of
+            if offset in self.ignore_if:
+                return
+
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
+                not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
+                     code[rtarget+3] == self.opc.POP_BLOCK and
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
+
+            # Does the "jump if" jump beyond a jump op?
+            # That is, we have something like:
+            #  JUMP_IF_FALSE HERE
+            #  ...
+            # JUMP_FORWARD
+            # HERE:
+            #
+            # If so, this can be block inside an "if" statement
+            # or a conditional assignment like:
+            #   x = 1 if x else 2
+            #
+            # There are other contexts we may need to consider
+            # like whether the target is "END_FINALLY"
+            # or if the condition jump is to a forward location
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
+
+                # If the jump target is back, we are looping
+                if (if_end < pre_rtarget and
+                    (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
+                    if (if_end > start):
+                        return
+
+                end = self.restrict_to_parent(if_end, parent)
+
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+
+                # if rtarget < end and (
+                #         code[rtarget] not in (self.opc.END_FINALLY,
+                #                               self.opc.JUMP_ABSOLUTE) and
+                #         code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
+                #                                         self.opc.END_FINALLY)):
+                #     self.structs.append({'type': 'else',
+                #                          'start': rtarget,
+                #                          'end': end})
+                #     self.else_start[rtarget] = end
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+                                       self.opc.BREAK_LOOP):
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': rtarget})
+                # It is important to distingish if this return is inside some sort
+                # except block return
+                jump_prev = prev_op[offset]
+                if self.is_pypy and code[jump_prev] == self.opc.COMPARE_OP:
+                    if self.opc.cmp_op[code[jump_prev+1]] == 'exception match':
+                        return
+                if self.version >= 3.5:
+                    # Python 3.5 may remove as dead code a JUMP
+                    # instruction after a RETURN_VALUE. So we check
+                    # based on seeing SETUP_EXCEPT various places.
+                    if code[rtarget] == self.opc.SETUP_EXCEPT:
+                        return
+                    # Check that next instruction after pops and jump is
+                    # not from SETUP_EXCEPT
+                    next_op = rtarget
+                    if code[next_op] == self.opc.POP_BLOCK:
+                        next_op += self.op_size(self.code[next_op])
+                    if code[next_op] == self.opc.JUMP_ABSOLUTE:
+                        next_op += self.op_size(self.code[next_op])
+                    if next_op in targets:
+                        for try_op in targets[next_op]:
+                            come_from_op = code[try_op]
+                            if come_from_op == self.opc.SETUP_EXCEPT:
+                                return
+                            pass
+                    pass
+                if code[pre_rtarget] == self.opc.RETURN_VALUE:
+                    self.return_end_ifs.add(pre_rtarget)
+                else:
+                    self.fixed_jumps[offset] = rtarget
+                    self.not_continue.add(pre_rtarget)
+
+
+        elif op == self.opc.SETUP_EXCEPT:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op == self.opc.SETUP_FINALLY:
+            target = self.get_target(offset)
+            end    = self.restrict_to_parent(target, parent)
+            self.fixed_jumps[offset] = end
+        elif op in self.jump_if_pop:
+            target = self.get_target(offset)
+            if target > offset:
+                unop_target = self.last_instr(offset, target, self.opc.JUMP_FORWARD, target)
+                if unop_target and code[unop_target+3] != self.opc.ROT_TWO:
+                    self.fixed_jumps[offset] = unop_target
+                else:
+                    self.fixed_jumps[offset] = self.restrict_to_parent(target, parent)
+                    pass
+                pass
+        elif self.version >= 3.5:
+            # 3.5+ has Jump optimization which too often causes RETURN_VALUE to get
+            # misclassified as RETURN_END_IF. Handle that here.
+            # In RETURN_VALUE, JUMP_ABSOLUTE, RETURN_VALUE is never RETURN_END_IF
+            if op == self.opc.RETURN_VALUE:
+                if (offset+1 < len(code) and code[offset+1] == self.opc.JUMP_ABSOLUTE and
+                    offset in self.return_end_ifs):
+                    self.return_end_ifs.remove(offset)
+                    pass
+                pass
+            elif op == self.opc.JUMP_FORWARD:
+                # If we have:
+                #   JUMP_FORWARD x, [non-jump, insns], RETURN_VALUE, x:
+                # then RETURN_VALUE is not RETURN_END_IF
+                rtarget = self.get_target(offset)
+                rtarget_prev = self.prev[rtarget]
+                if (code[rtarget_prev] == self.opc.RETURN_VALUE and
+                    rtarget_prev in self.return_end_ifs):
+                    i = rtarget_prev
+                    while i != offset:
+                        if code[i] in [opc.JUMP_FORWARD, opc.JUMP_ABSOLUTE]:
+                            return
+                        i = self.prev[i]
+                    self.return_end_ifs.remove(rtarget_prev)
+                pass
+        return
+
+    def rem_or(self, start, end, instr, target=None, include_beyond_target=False):
+        """
+        Find offsets of all requested <instr> between <start> and <end>,
+        optionally <target>ing specified offset, and return list found
+        <instr> offsets which are not within any POP_JUMP_IF_TRUE jumps.
+        """
+        assert(start>=0 and end<=len(self.code) and start <= end)
+
+        # Find all offsets of requested instructions
+        instr_offsets = self.all_instr(start, end, instr, target, include_beyond_target)
+        # Get all JUMP_IF_TRUE (or) offsets
+        pjit_offsets = self.all_instr(start, end, opc.JUMP_IF_TRUE)
+        filtered = []
+        for pjit_offset in pjit_offsets:
+            pjit_tgt = self.get_target(pjit_offset) - 3
+            for instr_offset in instr_offsets:
+                if instr_offset <= pjit_offset or instr_offset >= pjit_tgt:
+                    filtered.append(instr_offset)
+            instr_offsets = filtered
+            filtered = []
+        return instr_offsets
+
 if __name__ == "__main__":
     from uncompyle6 import PYTHON_VERSION
     if PYTHON_VERSION == 3.0:

uncompyle6/semantics/consts.py

@@ -0,0 +1,347 @@
+#  Copyright (c) 2017 by Rocky Bernstein
+"""Constants used in pysource.py"""
+
+import re, sys
+from uncompyle6.parsers.astnode import AST
+from uncompyle6 import PYTHON3
+from uncompyle6.scanners.tok import Token, NoneToken
+
+if PYTHON3:
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+
+
+LINE_LENGTH = 80
+
+# Some ASTs used for comparing code fragments (like 'return None' at
+# the end of functions).
+
+RETURN_LOCALS = AST('return_stmt',
+                    [ AST('ret_expr', [AST('expr', [ Token('LOAD_LOCALS') ])]),
+                      Token('RETURN_VALUE')])
+
+NONE = AST('expr', [ NoneToken ] )
+
+RETURN_NONE = AST('stmt',
+                  [ AST('return_stmt',
+                        [ NONE, Token('RETURN_VALUE')]) ])
+
+PASS = AST('stmts',
+           [ AST('sstmt',
+                 [ AST('stmt',
+                       [ AST('passstmt', [])])])])
+
+ASSIGN_DOC_STRING = lambda doc_string: \
+  AST('stmt',
+      [ AST('assign',
+            [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
+              AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
+            ])])
+
+NAME_MODULE = AST('stmt',
+                [ AST('assign',
+                    [ AST('expr',
+                          [Token('LOAD_NAME', pattr='__name__', offset=0, has_arg=True)]),
+                      AST('designator',
+                          [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)])
+                      ])])
+
+# God intended \t, but Python has decided to use 4 spaces.
+# If you want real tabs, use Go.
+# TAB = '\t'
+TAB = ' ' * 4
+INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
+
+TABLE_R = {
+    'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
+#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
+    'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
+#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
+}
+
+TABLE_R0 = {
+#    'BUILD_LIST':	( '[%C]',      (0,-1,', ') ),
+#    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
+#    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
+}
+TABLE_DIRECT = {
+    'BINARY_ADD':	( '+' ,),
+    'BINARY_SUBTRACT':	( '-' ,),
+    'BINARY_MULTIPLY':	( '*' ,),
+    'BINARY_DIVIDE':	( '/' ,),
+    'BINARY_MATRIX_MULTIPLY':	( '@' ,),
+    'BINARY_TRUE_DIVIDE':	( '/' ,),
+    'BINARY_FLOOR_DIVIDE':	( '//' ,),
+    'BINARY_MODULO':	( '%%',),
+    'BINARY_POWER':	( '**',),
+    'BINARY_LSHIFT':	( '<<',),
+    'BINARY_RSHIFT':	( '>>',),
+    'BINARY_AND':	( '&' ,),
+    'BINARY_OR':	( '|' ,),
+    'BINARY_XOR':	( '^' ,),
+    'INPLACE_ADD':	( '+=' ,),
+    'INPLACE_SUBTRACT':	( '-=' ,),
+    'INPLACE_MULTIPLY':	( '*=' ,),
+    'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
+    'INPLACE_DIVIDE':	( '/=' ,),
+    'INPLACE_TRUE_DIVIDE':	( '/=' ,),
+    'INPLACE_FLOOR_DIVIDE':	( '//=' ,),
+    'INPLACE_MODULO':	( '%%=',),
+    'INPLACE_POWER':	( '**=',),
+    'INPLACE_LSHIFT':	( '<<=',),
+    'INPLACE_RSHIFT':	( '>>=',),
+    'INPLACE_AND':	( '&=' ,),
+    'INPLACE_OR':	( '|=' ,),
+    'INPLACE_XOR':	( '^=' ,),
+    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
+
+    'UNARY_POSITIVE':	( '+',),
+    'UNARY_NEGATIVE':	( '-',),
+    'UNARY_INVERT':	( '~%c'),
+    'unary_expr':   ( '%c%c', 1, 0),
+
+    'unary_not':	( 'not %c', 0 ),
+    'unary_convert':	( '`%c`', 0 ),
+    'get_iter':	( 'iter(%c)', 0 ),
+    'slice0':		( '%c[:]', 0 ),
+    'slice1':		( '%c[%p:]', 0, (1, 100) ),
+    'slice2':		( '%c[:%p]', 0, (1, 100) ),
+    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
+
+    'IMPORT_FROM':	( '%{pattr}', ),
+    'load_attr':	( '%c.%[1]{pattr}', 0),
+    'LOAD_FAST':	( '%{pattr}', ),
+    'LOAD_NAME':	( '%{pattr}', ),
+    'LOAD_CLASSNAME':	( '%{pattr}', ),
+    'LOAD_GLOBAL':	( '%{pattr}', ),
+    'LOAD_DEREF':	( '%{pattr}', ),
+    'LOAD_LOCALS':	( 'locals()', ),
+    'LOAD_ASSERT':  ( '%{pattr}', ),
+#   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
+    'DELETE_FAST':	( '%|del %{pattr}\n', ),
+    'DELETE_NAME':	( '%|del %{pattr}\n', ),
+    'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
+    'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
+    'binary_subscr':	( '%c[%p]', 0, (1, 100)),
+    'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
+    'store_subscr':	( '%c[%c]', 0, 1),
+    'STORE_FAST':	( '%{pattr}', ),
+    'STORE_NAME':	( '%{pattr}', ),
+    'STORE_GLOBAL':	( '%{pattr}', ),
+    'STORE_DEREF':	( '%{pattr}', ),
+    'unpack':		( '%C%,', (1, maxint, ', ') ),
+
+    # This nonterminal we create on the fly in semantic routines
+    'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
+
+    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
+    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
+
+    # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
+    'list_iter':	( '%c', 0),
+    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
+    'list_if':		( ' if %c%c', 0, 2 ),
+    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
+    'lc_body':		( '', ),	# ignore when recusing
+
+    'comp_iter':	( '%c', 0),
+    'comp_if':		( ' if %c%c', 0, 2 ),
+    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
+    'comp_body':	( '', ),	# ignore when recusing
+    'set_comp_body':    ( '%c', 0 ),
+    'gen_comp_body':    ( '%c', 0 ),
+    'dict_comp_body':   ( '%c:%c', 1, 0 ),
+
+    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
+
+    # The 2nd parameter should have a = suffix.
+    # There is a rule with a 4th parameter "designator"
+    # which we don't use here.
+    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
+
+    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
+    'designList':	( '%c = %c', 0, -1 ),
+    'and':          	( '%c and %c', 0, 2 ),
+    'ret_and':        	( '%c and %c', 0, 2 ),
+    'and2':          	( '%c', 3 ),
+    'or':           	( '%c or %c', 0, 2 ),
+    'ret_or':           	( '%c or %c', 0, 2 ),
+    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
+    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
+    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
+    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
+    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
+    'return_lambda':    ('%c', 0),
+    'compare':		( '%p %[-1]{pattr} %p', (0, 19), (1, 19) ),
+    'cmp_list':		( '%p %p', (0, 20), (1, 19)),
+    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
+    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
+#   'classdef': 	(), # handled by n_classdef()
+    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
+    'funcdefdeco':  	( '\n\n%c', 0),
+    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
+    'mkfuncdeco0':  	( '%|def %c\n', 0),
+    'classdefdeco':  	( '\n\n%c', 0),
+    'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
+    'kwarg':    	( '%[0]{pattr}=%c', 1),
+    'kwargs':    	( '%D', (0, maxint, ', ') ),
+
+    'assert_expr_or': ( '%c or %c', 0, 2 ),
+    'assert_expr_and':    ( '%c and %c', 0, 2 ),
+    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
+    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
+    'print_item':  ( ', %c', 0),
+    'print_nl':	( '%|print\n', ),
+    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
+    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
+    'print_nl_to':	( '%|print >> %c\n', 0 ),
+    'print_to_items':	( '%C', (0, 2, ', ') ),
+
+    'call_stmt':	( '%|%p\n', (0, 200)),
+    'break_stmt':	( '%|break\n', ),
+    'continue_stmt':	( '%|continue\n', ),
+
+    'raise_stmt0':	( '%|raise\n', ),
+    'raise_stmt1':	( '%|raise %c\n', 0),
+    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
+#    'yield':	( 'yield %c', 0),
+#    'return_stmt':	( '%|return %c\n', 0),
+
+    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
+    'testtrue':         ( 'not %p', (0, 22) ),
+
+    'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtl':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelifstmt':	( '%|if %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifelifstmt':	( '%|elif %c:\n%+%c%-%c', 0, 1, 3 ),
+    'elifstmt':		( '%|elif %c:\n%+%c%-', 0, 1 ),
+    'elifelsestmt':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
+    'ifelsestmtr':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2 ),
+    'elifelsestmtr':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2 ),
+
+    'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
+    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
+    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
+    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
+    'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
+    'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
+    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
+    'forelsestmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'forelselaststmt':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
+    'forelselaststmtl':	(
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
+    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
+    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
+    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
+    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
+    'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
+    'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
+    'except':           ( '%|except:\n%+%c%-', 3 ),
+    'except_cond1':	( '%|except %c:\n', 1 ),
+    'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
+    'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
+    'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
+    'withasstmt':   ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
+    'passstmt':		( '%|pass\n', ),
+    'STORE_FAST':	( '%{pattr}', ),
+    'kv':		( '%c: %c', 3, 1 ),
+    'kv2':		( '%c: %c', 1, 2 ),
+    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
+    'importstmt': ( '%|import %c\n', 2),
+    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
+    'importstar': ( '%|from %[2]{pattr} import *\n', ),
+}
+
+
+MAP_DIRECT = (TABLE_DIRECT, )
+MAP_R0 = (TABLE_R0, -1, 0)
+MAP_R = (TABLE_R, -1)
+
+MAP = {
+    'stmt':		MAP_R,
+    'call_function':	MAP_R,
+    'del_stmt':		MAP_R,
+    'designator':	MAP_R,
+    'exprlist':		MAP_R0,
+}
+
+PRECEDENCE = {
+    'build_list':           0,
+    'mapexpr':              0,
+    'unary_convert':        0,
+    'dictcomp':             0,
+    'setcomp':              0,
+    'list_compr':           0,
+    'genexpr':              0,
+
+    'load_attr':            2,
+    'binary_subscr':        2,
+    'binary_subscr2':       2,
+    'slice0':               2,
+    'slice1':               2,
+    'slice2':               2,
+    'slice3':               2,
+    'buildslice2':          2,
+    'buildslice3':          2,
+    'call_function':        2,
+
+    'BINARY_POWER':         4,
+
+    'unary_expr':           6,
+
+    'BINARY_MULTIPLY':      8,
+    'BINARY_DIVIDE':        8,
+    'BINARY_TRUE_DIVIDE':   8,
+    'BINARY_FLOOR_DIVIDE':  8,
+    'BINARY_MODULO':        8,
+
+    'BINARY_ADD':           10,
+    'BINARY_SUBTRACT':      10,
+
+    'BINARY_LSHIFT':        12,
+    'BINARY_RSHIFT':        12,
+
+    'BINARY_AND':           14,
+
+    'BINARY_XOR':           16,
+
+    'BINARY_OR':            18,
+
+    'cmp':                  20,
+
+    'unary_not':            22,
+
+    'and':                  24,
+    'ret_and':              24,
+
+    'or':                   26,
+    'ret_or':               26,
+
+    'conditional':          28,
+    'conditionalnot':       28,
+    'ret_cond':             28,
+    'ret_cond_not':         28,
+
+    '_mklambda':            30,
+    'yield':                101,
+    'yield_from':           101
+}
+
+ASSIGN_TUPLE_PARAM = lambda param_name: \
+             AST('expr', [ Token('LOAD_FAST', pattr=param_name) ])
+
+escape = re.compile(r'''
+            (?P<prefix> [^%]* )
+            % ( \[ (?P<child> -? \d+ ) \] )?
+                ((?P<type> [^{] ) |
+                 ( [{] (?P<expr> [^}]* ) [}] ))
+        ''', re.VERBOSE)

uncompyle6/semantics/fragments.py

@@ -67,17 +67,18 @@ from uncompyle6.show import (
     maybe_show_ast_param_default,
 )
 
-from uncompyle6.semantics.pysource import AST, INDENT_PER_LEVEL, NONE, PRECEDENCE, \
-     ParserError, TABLE_DIRECT, escape, find_globals, minint, MAP
+from uncompyle6.parsers.astnode import AST
+
+from uncompyle6.semantics.pysource import (
+    ParserError, find_globals, StringIO)
+
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, NONE, PRECEDENCE,
+    TABLE_DIRECT, escape, minint, MAP
+    )
 
 from uncompyle6.semantics.make_function import find_all_globals, find_none
 
-if PYTHON3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
-
-
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
 if PYTHON_VERSION < 2.6:

uncompyle6/semantics/make_function.py

@@ -66,7 +66,11 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     # MAKE_FUNCTION_... or MAKE_CLOSURE_...
     assert node[-1].type.startswith('MAKE_')
 
+    annotate_tuple = None
+    for annotate_last in range(len(node)-1, -1, -1):
+        if node[annotate_last] == 'annotate_tuple':
             annotate_tuple = node[annotate_last]
+            break
     annotate_args = {}
 
     if (annotate_tuple == 'annotate_tuple'
@@ -143,13 +147,19 @@ def make_function3_annotate(self, node, isLambda, nested=1,
     suffix = ''
     for param in paramnames[:i]:
         self.write(suffix, param)
-        if param in annotate_args:
-            value, string = annotate_args[param]
-            if string:
-                self.write(': "%s"' % value)
-            else:
-                self.write(': %s' % value)
         suffix = ', '
+        if param in annotate_tuple[0].attr:
+            p = annotate_tuple[0].attr.index(param)
+            self.write(': ')
+            self.preorder(node[p])
+            if (line_number != self.line_number):
+                suffix = ",\n" + indent
+                line_number = self.line_number
+            # value, string = annotate_args[param]
+            # if string:
+            #     self.write(': "%s"' % value)
+            # else:
+            #     self.write(': %s' % value)
 
     if i > 0:
         suffix = ', '
@@ -161,7 +171,10 @@ def make_function3_annotate(self, node, isLambda, nested=1,
             param = paramnames[i]
             self.write(param)
             if param in annotate_args:
-                self.write(':"%s' % annotate_args[param])
+                aa = annotate_args[param]
+                if isinstance(aa, tuple):
+                    aa = aa[0]
+                self.write(': "%s"' % aa)
             self.write('=')
             i += 1
             self.preorder(n)
@@ -187,6 +200,9 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         i = 0
         for n in node[0]:
             if n == 'kwarg':
+                if (line_number != self.line_number):
+                    self.write("\n" + indent)
+                    line_number = self.line_number
                 self.write('%s=' % n[0].pattr)
                 self.preorder(n[1])
                 if i < last:
@@ -205,19 +221,24 @@ def make_function3_annotate(self, node, isLambda, nested=1,
         self.write(": ")
     else:
         self.write(')')
-        if 'return' in annotate_args:
-            value, string = annotate_args['return']
-            if string:
-                self.write(' -> "%s"' % value)
-            else:
-                self.write(' -> %s' % value)
+        if 'return' in annotate_tuple[0].attr:
+            if (line_number != self.line_number):
+                self.write("\n" + indent)
+                line_number = self.line_number
+            self.write(' -> ')
+            # value, string = annotate_args['return']
+            # if string:
+            #     self.write(' -> "%s"' % value)
+            # else:
+            #     self.write(' -> %s' % value)
+            self.preorder(node[annotate_last-1])
 
         self.println(":")
 
     if (len(code.co_consts) > 0 and
         code.co_consts[0] is not None and not isLambda): # ugly
         # docstring exists, dump it
-        print_docstring(self, indent, code.co_consts[0])
+        print_docstring(self, self.indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'

uncompyle6/semantics/pysource.py

@@ -67,21 +67,30 @@ methods implement most of the below.
   The '%' may optionally be followed by a number (C) in square brackets, which
   makes the engine walk down to N[C] before evaluating the escape code.
 """
-import sys, re
+import sys
 
 from uncompyle6 import PYTHON3
 from xdis.code import iscode
+from xdis.util import COMPILER_FLAG_BIT
+
 from uncompyle6.parser import get_python_parser
 from uncompyle6.parsers.astnode import AST
 from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.scanner import Code, get_scanner
-from uncompyle6.scanners.tok import Token, NoneToken
 import uncompyle6.parser as python_parser
 from uncompyle6.semantics.make_function import (
     make_function2, make_function3, make_function3_annotate, find_globals)
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.check_ast import checker
 from uncompyle6.semantics.helper import print_docstring
+from uncompyle6.scanners.tok import Token
+
+from uncompyle6.semantics.consts import (
+    LINE_LENGTH, RETURN_LOCALS, NONE, RETURN_NONE, PASS,
+    ASSIGN_DOC_STRING, NAME_MODULE, TAB,
+    INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT, MAP_DIRECT,
+    MAP, PRECEDENCE, ASSIGN_TUPLE_PARAM, escape, maxint, minint)
+
 
 from uncompyle6.show import (
     maybe_show_ast,
@@ -89,348 +98,8 @@ from uncompyle6.show import (
 
 if PYTHON3:
     from io import StringIO
-    minint = -sys.maxsize-1
-    maxint = sys.maxsize
 else:
     from StringIO import StringIO
-    minint = -sys.maxint-1
-    maxint = sys.maxint
-
-LINE_LENGTH = 80
-
-# Some ASTs used for comparing code fragments (like 'return None' at
-# the end of functions).
-
-RETURN_LOCALS = AST('return_stmt',
-                    [ AST('ret_expr', [AST('expr', [ Token('LOAD_LOCALS') ])]),
-                      Token('RETURN_VALUE')])
-
-NONE = AST('expr', [ NoneToken ] )
-
-RETURN_NONE = AST('stmt',
-                  [ AST('return_stmt',
-                        [ NONE, Token('RETURN_VALUE')]) ])
-
-PASS = AST('stmts',
-           [ AST('sstmt',
-                 [ AST('stmt',
-                       [ AST('passstmt', [])])])])
-
-ASSIGN_DOC_STRING = lambda doc_string: \
-  AST('stmt',
-      [ AST('assign',
-            [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
-              AST('designator', [ Token('STORE_NAME', pattr='__doc__')])
-            ])])
-
-BUILD_TUPLE_0 = AST('expr',
-                    [ AST('build_list',
-                          [ Token('BUILD_TUPLE_0') ])])
-
-NAME_MODULE = AST('stmt',
-                [ AST('assign',
-                    [ AST('expr',
-                          [Token('LOAD_NAME', pattr='__name__', offset=0, has_arg=True)]),
-                      AST('designator',
-                          [ Token('STORE_NAME', pattr='__module__', offset=3, has_arg=True)])
-                      ])])
-
-# God intended \t, but Python has decided to use 4 spaces.
-# If you want real tabs, use Go.
-# TAB = '\t'
-TAB = ' ' * 4
-INDENT_PER_LEVEL = ' ' # additional intent per pretty-print level
-
-TABLE_R = {
-    'STORE_ATTR':	( '%c.%[1]{pattr}', 0),
-#   'STORE_SUBSCR':	( '%c[%c]', 0, 1 ),
-    'DELETE_ATTR':	( '%|del %c.%[-1]{pattr}\n', 0 ),
-#   'EXEC_STMT':	( '%|exec %c in %[1]C\n', 0, (0,maxint,', ') ),
-}
-
-TABLE_R0 = {
-#    'BUILD_LIST':	( '[%C]',      (0,-1,', ') ),
-#    'BUILD_TUPLE':	( '(%C)',      (0,-1,', ') ),
-#    'CALL_FUNCTION':	( '%c(%P)', 0, (1,-1,', ') ),
-}
-TABLE_DIRECT = {
-    'BINARY_ADD':	( '+' ,),
-    'BINARY_SUBTRACT':	( '-' ,),
-    'BINARY_MULTIPLY':	( '*' ,),
-    'BINARY_DIVIDE':	( '/' ,),
-    'BINARY_MATRIX_MULTIPLY':	( '@' ,),
-    'BINARY_TRUE_DIVIDE':	( '/' ,),
-    'BINARY_FLOOR_DIVIDE':	( '//' ,),
-    'BINARY_MODULO':	( '%%',),
-    'BINARY_POWER':	( '**',),
-    'BINARY_LSHIFT':	( '<<',),
-    'BINARY_RSHIFT':	( '>>',),
-    'BINARY_AND':	( '&' ,),
-    'BINARY_OR':	( '|' ,),
-    'BINARY_XOR':	( '^' ,),
-    'INPLACE_ADD':	( '+=' ,),
-    'INPLACE_SUBTRACT':	( '-=' ,),
-    'INPLACE_MULTIPLY':	( '*=' ,),
-    'INPLACE_MATRIX_MULTIPLY':	( '@=' ,),
-    'INPLACE_DIVIDE':	( '/=' ,),
-    'INPLACE_TRUE_DIVIDE':	( '/=' ,),
-    'INPLACE_FLOOR_DIVIDE':	( '//=' ,),
-    'INPLACE_MODULO':	( '%%=',),
-    'INPLACE_POWER':	( '**=',),
-    'INPLACE_LSHIFT':	( '<<=',),
-    'INPLACE_RSHIFT':	( '>>=',),
-    'INPLACE_AND':	( '&=' ,),
-    'INPLACE_OR':	( '|=' ,),
-    'INPLACE_XOR':	( '^=' ,),
-    'binary_expr':	( '%c %c %c', 0, -1, 1 ),
-
-    'UNARY_POSITIVE':	( '+',),
-    'UNARY_NEGATIVE':	( '-',),
-    'UNARY_INVERT':	( '~%c'),
-    'unary_expr':   ( '%c%c', 1, 0),
-
-    'unary_not':	( 'not %c', 0 ),
-    'unary_convert':	( '`%c`', 0 ),
-    'get_iter':	( 'iter(%c)', 0 ),
-    'slice0':		( '%c[:]', 0 ),
-    'slice1':		( '%c[%p:]', 0, (1, 100) ),
-    'slice2':		( '%c[:%p]', 0, (1, 100) ),
-    'slice3':		( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
-
-    'IMPORT_FROM':	( '%{pattr}', ),
-    'load_attr':	( '%c.%[1]{pattr}', 0),
-    'LOAD_FAST':	( '%{pattr}', ),
-    'LOAD_NAME':	( '%{pattr}', ),
-    'LOAD_CLASSNAME':	( '%{pattr}', ),
-    'LOAD_GLOBAL':	( '%{pattr}', ),
-    'LOAD_DEREF':	( '%{pattr}', ),
-    'LOAD_LOCALS':	( 'locals()', ),
-    'LOAD_ASSERT':  ( '%{pattr}', ),
-#   'LOAD_CONST':	( '%{pattr}', ),	# handled by n_LOAD_CONST
-    'DELETE_FAST':	( '%|del %{pattr}\n', ),
-    'DELETE_NAME':	( '%|del %{pattr}\n', ),
-    'DELETE_GLOBAL':	( '%|del %{pattr}\n', ),
-    'delete_subscr':	( '%|del %c[%c]\n', 0, 1,),
-    'binary_subscr':	( '%c[%p]', 0, (1, 100)),
-    'binary_subscr2':	( '%c[%p]', 0, (1, 100)),
-    'store_subscr':	( '%c[%c]', 0, 1),
-    'STORE_FAST':	( '%{pattr}', ),
-    'STORE_NAME':	( '%{pattr}', ),
-    'STORE_GLOBAL':	( '%{pattr}', ),
-    'STORE_DEREF':	( '%{pattr}', ),
-    'unpack':		( '%C%,', (1, maxint, ', ') ),
-
-    # This nonterminal we create on the fly in semantic routines
-    'unpack_w_parens':	( '(%C%,)', (1, maxint, ', ') ),
-
-    'unpack_list':	( '[%C]', (1, maxint, ', ') ),
-    'build_tuple2':	( '%P', (0, -1, ', ', 100) ),
-
-    # 'list_compr':	( '[ %c ]', -2),	# handled by n_list_compr
-    'list_iter':	( '%c', 0),
-    'list_for':		( ' for %c in %c%c', 2, 0, 3 ),
-    'list_if':		( ' if %c%c', 0, 2 ),
-    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
-    'lc_body':		( '', ),	# ignore when recusing
-
-    'comp_iter':	( '%c', 0),
-    'comp_if':		( ' if %c%c', 0, 2 ),
-    'comp_ifnot':	( ' if not %p%c', (0, 22), 2 ),
-    'comp_body':	( '', ),	# ignore when recusing
-    'set_comp_body':    ( '%c', 0 ),
-    'gen_comp_body':    ( '%c', 0 ),
-    'dict_comp_body':   ( '%c:%c', 1, 0 ),
-
-    'assign':		( '%|%c = %p\n', -1, (0, 200) ),
-
-    # The 2nd parameter should have a = suffix.
-    # There is a rule with a 4th parameter "designator"
-    # which we don't use here.
-    'augassign1':	( '%|%c %c %c\n', 0, 2, 1),
-
-    'augassign2':	( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4),
-    'designList':	( '%c = %c', 0, -1 ),
-    'and':          	( '%c and %c', 0, 2 ),
-    'ret_and':        	( '%c and %c', 0, 2 ),
-    'and2':          	( '%c', 3 ),
-    'or':           	( '%c or %c', 0, 2 ),
-    'ret_or':           	( '%c or %c', 0, 2 ),
-    'conditional':  ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)),
-    'ret_cond':     ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)),
-    'conditionalnot':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)),
-    'ret_cond_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)),
-    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
-    'return_lambda':    ('%c', 0),
-    'compare':		( '%p %[-1]{pattr} %p', (0, 19), (1, 19) ),
-    'cmp_list':		( '%p %p', (0, 20), (1, 19)),
-    'cmp_list1':	( '%[3]{pattr} %p %p', (0, 19), (-2, 19)),
-    'cmp_list2':	( '%[1]{pattr} %p', (0, 19)),
-#   'classdef': 	(), # handled by n_classdef()
-    'funcdef':  	( '\n\n%|def %c\n', -2), # -2 to handle closures
-    'funcdefdeco':  	( '\n\n%c', 0),
-    'mkfuncdeco':  	( '%|@%c\n%c', 0, 1),
-    'mkfuncdeco0':  	( '%|def %c\n', 0),
-    'classdefdeco':  	( '\n\n%c', 0),
-    'classdefdeco1':  	( '%|@%c\n%c', 0, 1),
-    'kwarg':    	( '%[0]{pattr}=%c', 1),
-    'kwargs':    	( '%D', (0, maxint, ', ') ),
-
-    'assert_expr_or': ( '%c or %c', 0, 2 ),
-    'assert_expr_and':    ( '%c and %c', 0, 2 ),
-    'print_items_stmt': ( '%|print %c%c,\n', 0, 2),  # Python 2 only
-    'print_items_nl_stmt': ( '%|print %c%c\n', 0, 2),
-    'print_item':  ( ', %c', 0),
-    'print_nl':	( '%|print\n', ),
-    'print_to':		( '%|print >> %c, %c,\n', 0, 1 ),
-    'print_to_nl':	( '%|print >> %c, %c\n', 0, 1 ),
-    'print_nl_to':	( '%|print >> %c\n', 0 ),
-    'print_to_items':	( '%C', (0, 2, ', ') ),
-
-    'call_stmt':	( '%|%p\n', (0, 200)),
-    'break_stmt':	( '%|break\n', ),
-    'continue_stmt':	( '%|continue\n', ),
-
-    'raise_stmt0':	( '%|raise\n', ),
-    'raise_stmt1':	( '%|raise %c\n', 0),
-    'raise_stmt3':	( '%|raise %c, %c, %c\n', 0, 1, 2),
-#    'yield':	( 'yield %c', 0),
-#    'return_stmt':	( '%|return %c\n', 0),
-
-    'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'testtrue':         ( 'not %p', (0, 22) ),
-
-    'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelsestmtl':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelifstmt':	( '%|if %c:\n%+%c%-%c', 0, 1, 3 ),
-    'elifelifstmt':	( '%|elif %c:\n%+%c%-%c', 0, 1, 3 ),
-    'elifstmt':		( '%|elif %c:\n%+%c%-', 0, 1 ),
-    'elifelsestmt':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
-    'ifelsestmtr':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 2 ),
-    'elifelsestmtr':	( '%|elif %c:\n%+%c%-%|else:\n%+%c%-\n\n', 0, 1, 2 ),
-
-    'whileTruestmt':	( '%|while True:\n%+%c%-\n\n', 1 ),
-    'whilestmt':	( '%|while %c:\n%+%c%-\n\n', 1, 2 ),
-    'while1stmt':	( '%|while 1:\n%+%c%-\n\n', 1 ),
-    'while1elsestmt':  ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
-    'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
-    'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
-    'forstmt':		( '%|for %c in %c:\n%+%c%-\n\n', 3, 1, 4 ),
-    'forelsestmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
-    'forelselaststmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', 3, 1, 4, -2),
-    'forelselaststmtl':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 3, 1, 4, -2),
-    'trystmt':		( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
-    'tryelsestmt':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
-    'tryelsestmtc':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
-    'tryelsestmtl':	( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),
-    'tf_trystmt':	( '%c%-%c%+', 1, 3 ),
-    'tf_tryelsestmt':	( '%c%-%c%|else:\n%+%c', 1, 3, 4 ),
-    'tryfinallystmt':	( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 5 ),
-    'except':           ( '%|except:\n%+%c%-', 3 ),
-    'except_cond1':	( '%|except %c:\n', 1 ),
-    'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
-    'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
-    'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
-    'withasstmt':   ( '%|with %c as %c:\n%+%c%-', 0, 2, 3),
-    'passstmt':		( '%|pass\n', ),
-    'STORE_FAST':	( '%{pattr}', ),
-    'kv':		( '%c: %c', 3, 1 ),
-    'kv2':		( '%c: %c', 1, 2 ),
-    'mapexpr':		( '{%[1]C}', (0, maxint, ', ') ),
-    'importstmt': ( '%|import %c\n', 2),
-    'importfrom': ( '%|from %[2]{pattr} import %c\n', 3 ),
-    'importstar': ( '%|from %[2]{pattr} import *\n', ),
-}
-
-
-MAP_DIRECT = (TABLE_DIRECT, )
-MAP_R0 = (TABLE_R0, -1, 0)
-MAP_R = (TABLE_R, -1)
-
-MAP = {
-    'stmt':		MAP_R,
-    'call_function':	MAP_R,
-    'del_stmt':		MAP_R,
-    'designator':	MAP_R,
-    'exprlist':		MAP_R0,
-}
-
-PRECEDENCE = {
-    'build_list':           0,
-    'mapexpr':              0,
-    'unary_convert':        0,
-    'dictcomp':             0,
-    'setcomp':              0,
-    'list_compr':           0,
-    'genexpr':              0,
-
-    'load_attr':            2,
-    'binary_subscr':        2,
-    'binary_subscr2':       2,
-    'slice0':               2,
-    'slice1':               2,
-    'slice2':               2,
-    'slice3':               2,
-    'buildslice2':          2,
-    'buildslice3':          2,
-    'call_function':        2,
-
-    'BINARY_POWER':         4,
-
-    'unary_expr':           6,
-
-    'BINARY_MULTIPLY':      8,
-    'BINARY_DIVIDE':        8,
-    'BINARY_TRUE_DIVIDE':   8,
-    'BINARY_FLOOR_DIVIDE':  8,
-    'BINARY_MODULO':        8,
-
-    'BINARY_ADD':           10,
-    'BINARY_SUBTRACT':      10,
-
-    'BINARY_LSHIFT':        12,
-    'BINARY_RSHIFT':        12,
-
-    'BINARY_AND':           14,
-
-    'BINARY_XOR':           16,
-
-    'BINARY_OR':            18,
-
-    'cmp':                  20,
-
-    'unary_not':            22,
-
-    'and':                  24,
-    'ret_and':              24,
-
-    'or':                   26,
-    'ret_or':               26,
-
-    'conditional':          28,
-    'conditionalnot':       28,
-    'ret_cond':             28,
-    'ret_cond_not':         28,
-
-    '_mklambda':            30,
-    'yield':                101,
-    'yield_from':           101
-}
-
-ASSIGN_TUPLE_PARAM = lambda param_name: \
-             AST('expr', [ Token('LOAD_FAST', pattr=param_name) ])
-
-escape = re.compile(r'''
-            (?P<prefix> [^%]* )
-            % ( \[ (?P<child> -? \d+ ) \] )?
-                ((?P<type> [^{] ) |
-                 ( [{] (?P<expr> [^}]* ) [}] ))
-        ''', re.VERBOSE)
 
 def is_docstring(node):
     try:
@@ -622,10 +291,9 @@ class SourceWalker(GenericASTTraversal, object):
                     code = node[-3]
 
                 self.indentMore()
-                if self.version == 3.1:
-                    annotate_last = -4
-                else:
-                    annotate_last = -5
+                for annotate_last in range(len(node)-1, -1, -1):
+                    if node[annotate_last] == 'annotate_tuple':
+                        break
 
                 # FIXME: handle and pass full annotate args
                 make_function3_annotate(self, node, isLambda=False,
@@ -646,7 +314,42 @@ class SourceWalker(GenericASTTraversal, object):
                 TABLE_DIRECT.update({
                     'LOAD_CLASSDEREF':	( '%{pattr}', ),
                     })
+                ########################
+                # Python 3.5+ Additions
+                #######################
                 if version >= 3.5:
+                    TABLE_DIRECT.update({
+                        'await_stmt':	       ( '%|await %c', 0),
+                        'async_for_stmt':      (
+                            '%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
+                        'async_forelse_stmt':  (
+                            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
+                        'async_with_stmt':     (
+                            '%|async with %c:\n%+%c%-', 0, 7),
+                        'async_with_as_stmt':  (
+                            '%|async with %c as %c:\n%+%c%-', 0, 6, 7),
+
+                    })
+                    def n_async_call_function(node):
+                        self.f.write('async ')
+                        node.type == 'call_function'
+                        p = self.prec
+                        self.prec = 80
+                        self.engine(('%c(%P)', 0, (1, -4, ', ', 100)), node)
+                        self.prec = p
+                        self.prune()
+                    self.n_async_call_function = n_async_call_function
+
+                    def n_funcdef(node):
+                        code_node = node[0][1]
+                        if (code_node == 'LOAD_CONST' and iscode(code_node.attr)
+                            and code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE']):
+                            self.engine(('\n\n%|async def %c\n', -2), node)
+                        else:
+                            self.engine(('\n\n%|def %c\n', -2), node)
+                        self.prune()
+                    self.n_funcdef = n_funcdef
+
                     def n_unmapexpr(node):
                         last_n = node[0][-1]
                         for n in node[0]:
@@ -1809,14 +1512,20 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec = 100
         lastnode = node.pop()
         lastnodetype = lastnode.type
+
+        # If this build list is inside a CALL_FUNCTION_VAR,
+        # then the first * has already been printed.
+        # Until I have a better way to check for CALL_FUNCTION_VAR,
+        # will assume that if the text ends in *.
+        last_was_star = self.f.getvalue().endswith('*')
+
         have_star = False
-        if lastnodetype.startswith('BUILD_LIST'):
-            # 3.5+ has BUILD_LIST_UNPACK
-            if lastnodetype == 'BUILD_LIST_UNPACK':
+        if lastnodetype.endswith('UNPACK'):
             # FIXME: need to handle range of BUILD_LIST_UNPACK
             have_star = True
             endchar = ''
         else:
+            if lastnodetype.startswith('BUILD_LIST'):
                 self.write('['); endchar = ']'
             elif lastnodetype.startswith('BUILD_TUPLE'):
                 self.write('('); endchar = ')'
@@ -1852,8 +1561,13 @@ class SourceWalker(GenericASTTraversal, object):
                 sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
             else:
                 if sep != '': sep += ' '
+            if not last_was_star:
                 if have_star:
                     sep += '*'
+                    pass
+                pass
+            else:
+                last_was_star = False
             self.write(sep, value)
             sep = ','
         if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
@@ -1915,8 +1629,7 @@ class SourceWalker(GenericASTTraversal, object):
         beginning of this module for the how we interpret format specifications such as
         %c, %C, and so on.
         """
-
-        # self.println("----> ", startnode.type)
+        # self.println("----> ", startnode.type, ', ', entry[0])
         fmt = entry[0]
         arg = 1
         i = 0