Administrivia Add 2.6 back into older dist

Allow format specifier "%p" to indicate a nonterminal name,
like "%c" allows.

store_subscr -> store_subscript to match Python AST a little closer.

NEWS.md

@@ -1,4 +1,33 @@
-3.3.0 2019-03-23 Holy Week
+3.3.2 2019-05-03 Better Friday
+==============================
+
+As before, lots of decomplation bugs fixed. The focus has primarily
+been on Python 3.6. We can now parse the entire 3.6.8 Python library
+and verify that without an error. The same is true for 3.5.8. A number
+of the bugs fixed though are not contained to these versions. In fact
+some span back as far as 2.x
+
+But as before, many more remain in the 3.7 and 3.8 range which will
+get addressed in future releases
+
+Pypy 3.6 support was started. Pypy 3.x detection fixed (via xdis)
+
+
+3.3.1 2019-04-19 Good Friday
+==========================
+
+Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry though, many more remain.
+
+* Add annotation return values in 3.6+
+* Fix 3.6+ lambda parameter handling decompilation
+* Fix 3.7+ chained comparision decompilation
+* split out semantic-action customization into more separate files
+* Add 3.8 try/else
+* Fix 2.7 generator decompilation
+* Fix some parser failures fixes in 3.4+ using test_pyenvlib
+* Add more run tests
+
+3.3.0 2019-43-14 Holy Week
 ==========================
 
 * First cut at Python 3.8 (many bug remain)

README.rst

@@ -12,7 +12,7 @@ Introduction
 
 *uncompyle6* translates Python bytecode back into equivalent Python
 source code. It accepts bytecodes from Python version 1.3 to version
-3.7, spanning over 22 years of Python releases. We include Dropbox's
+3.8, spanning over 24 years of Python releases. We include Dropbox's
 Python 2.5 bytecode and some PyPy bytecode.
 
 Why this?
@@ -76,7 +76,7 @@ Requirements
 The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
 or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
 python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.6 and the
+Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.8 and the
 above-mentioned PyPy versions.
 
 Installation

__pkginfo__.py

@@ -43,6 +43,7 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 3.5',
                 'Programming Language :: Python :: 3.6',
                 'Programming Language :: Python :: 3.7',
+                'Programming Language :: Python :: 3.8',
                 'Topic :: Software Development :: Debuggers',
                 'Topic :: Software Development :: Libraries :: Python Modules',
                 ]
@@ -57,7 +58,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
-                      'xdis >= 4.0.0, < 4.1.0']
+                      'xdis >= 4.0.1, < 4.1.0']
 
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.2.6 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.6.8 3.7.3 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.8'

admin-tools/pyenv-older-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6'
+export PYVERSIONS='2.4.6 2.5.6 2.6.9'

pytest/test_pysource.py

@@ -123,11 +123,17 @@ def test_tables():
                             "Full entry: %s" %
                             (name, k, arg, typ, entry[arg], type(entry[arg]), entry)
                             )
-                        assert len(tup) == 2
+                        assert 2 <= len(tup) <= 3
                         for j, x in enumerate(tup):
-                            assert isinstance(x, int), (
-                                "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
-                                (name, k, arg, j, typ, x, type(x), entry)
+                            if len(tup) == 3 and j == 1:
+                                assert isinstance(x, str), (
+                                    "%s[%s][%d][%d] type '%s' is '%s should be an string but is %s. Full entry: %s" %
+                                    (name, k, arg, j, typ, x, type(x), entry)
+                                    )
+                            else:
+                                assert isinstance(x, int), (
+                                    "%s[%s][%d][%d] type '%s' is '%s should be an int but is %s. Full entry: %s" %
+                                    (name, k, arg, j, typ, x, type(x), entry)
                                 )
                         pass
                     arg += 1

test/Makefile

@@ -264,8 +264,8 @@ check-bytecode-3.5:
 
 #: Check deparsing Python 3.6
 check-bytecode-3.6:
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify
 
 #: Check deparsing Python 3.7
 check-bytecode-3.7:
@@ -273,6 +273,7 @@ check-bytecode-3.7:
 
 #: Check deparsing Python 3.8
 check-bytecode-3.8:
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify
 
 #: short tests for bytecodes only for this version of Python
@@ -312,6 +313,12 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
+#: PyPy 5.0.x with Python 3.6 ...
+7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+
+
 clean: clean-py-dis clean-dis clean-unverified
 
 clean-dis:

test/simple_source/bug30/04_and_del.py

@@ -0,0 +1,20 @@
+# From 2.5.6 osxemxpath.py
+# Bug is in getting "and" and "del" correct
+def normpath(comps):
+    i = 0
+    while i < len(comps):
+        if comps[i] == '.':
+            del comps[i]
+        elif comps[i] == '..' and i > 0 and comps[i-1] not in ('', '..'):
+            del comps[i-1:i+1]
+            i = i - 1
+        elif comps[i] == '' and i > 0 and comps[i-1] != '':
+            del comps[i]
+        else:
+            i = i + 1
+    return comps
+
+assert normpath(['.']) == []
+assert normpath(['a', 'b', '..']) == ['a']
+assert normpath(['a', 'b', '', 'c']) == ['a', 'b', 'c']
+assert normpath(['a', 'b', '.', '', 'c', '..']) == ['a', 'b']

test/simple_source/bug31/02_ifelse_comprehension.py

@@ -1,12 +1,18 @@
-# Python 2.7 sqlalchemy-1.013/sql/crud.py
-def _extend_values_for_multiparams(compiler, stmt, c):
-    c(
-        [
-            (
-                (compiler() if compiler()
-                    else compiler())
-                if c in stmt else compiler(),
-            )
-        ]
-        for i in enumerate(stmt)
+# Adapted from Python 2.7 sqlalchemy-1.013/sql/crud.py
+# Bug was in handling generator comprehension
+# In 2.6, 2.7 JUMP_ABSOLUTEs rather than JUMP_BACKs are generated
+
+# RUNNABLE!
+def extend(stmt, a, c, c1, c2, c3):
+    return c(
+        ([ (5 if c1 else c2)
+          if a else c3
+        ] for i in enumerate(stmt))
     )
+
+def foo(gen):
+    return list(gen)
+
+assert extend([0], 0, foo, True, 'c2', 'c3') == [['c3']]
+assert extend([0, 1], 1, foo, False, 'c2', 'c3') == [['c2'], ['c2']]
+assert extend([0, 1], False, foo, False, 'c2', 'c3') == [['c3'], ['c3']]

test/simple_source/bug33/01_triple_compare.py

@@ -1,8 +1,11 @@
 # In Python 3.3+ this uses grammar rule
 # compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
+# In Python 3.6 uses this uses grammar rule
+# compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
 
 # Seen in Python 3.3 ipaddress.py
 
+# RUNNABLE!
 def _is_valid_netmask(netmask):
     return 0 <= netmask <= 10
 
@@ -10,4 +13,4 @@ def _is_valid_netmask(netmask):
 # detections
 
 # See in 2.6.9 quopri.py ishex():
-'0' <= __file__ <= '9' or 'a' <= __file__ <= 'f' or 'A' <= __file__ <= 'F'
+assert not '0' <= __file__ <= '9' or 'a' <= __file__ <= 'f' or 'A' <= __file__ <= 'F'

test/simple_source/bug33/02_pos_args.py

@@ -1,6 +1,12 @@
 # From Python 3.3.6 hmac.py
-# Problem was getting wrong placement of positional args
+# Problem was getting wrong placement of positional args.
+# In 3.6+ paramter handling changes
+
+# RUNNABLE!
+
 digest_cons = lambda d=b'': 5
 
 # Handle single kwarg
-lambda *, d=0: None
+x = lambda *, d=0: d
+assert x(d=1) == 1
+assert x() == 0

test/simple_source/bug33/08_if_else.py

@@ -0,0 +1,10 @@
+# From python 3.3.7 trace
+# Bug was not having not having semantic rule for conditional not
+
+# RUNNABLE!
+def init(modules=None):
+    mods = set() if not modules else set(modules)
+    return mods
+
+assert init() == set()
+assert init([1, 2, 3]) == set([1, 2, 3])

test/simple_source/bug35/06_while_return.py

@@ -1,28 +1,32 @@
 # From Python 3.4 asynchat.py
 # Tests presence or absense of
 # SETUP_LOOP testexpr return_stmts POP_BLOCK COME_FROM_LOOP
+# Note: that there is no JUMP_BACK because of the return_stmts.
 
-def initiate_send(self, num_sent, first):
-    while self.producer_fifo and self.connected:
+def initiate_send(a, b, c, num_sent):
+    while a and b:
         try:
-            5
-        except OSError:
-            return
+            1 / (b - 1)
+        except ZeroDivisionError:
+            return 1
 
         if num_sent:
-            if first:
-                self.producer_fifo = '6'
-            else:
-                del self.producer_fifo[0]
-        return
+            c = 2
+        return c
 
 
-# FIXME: this causes a parse error:
-# def initiate_send(self):
-#     while self.producer_fifo and self.connected:
-#         try:
-#             6
-#         except OSError:
-#             return
+def initiate_send2(a, b):
+    while a and b:
+        try:
+            1 / (b - 1)
+        except ZeroDivisionError:
+            return 1
 
-#         return
+        return 2
+
+assert initiate_send(1, 1, 2, False) == 1
+assert initiate_send(1, 2, 3, False) == 3
+assert initiate_send(1, 2, 3, True) == 2
+
+assert initiate_send2(1, 1) == 1
+assert initiate_send2(1, 2) == 2

test/simple_source/bug36/01_fstring.py

@@ -17,3 +17,25 @@ x = f"{k}={v!r}"
 y = f"functools.{x}({', '.join(v)})"
 assert x == "1=['2']"
 assert y == "functools.1=['2'](2)"
+
+# From 3.6 http/client.py
+# Bug is in handling  X
+chunk = ['a', 'b', 'c']
+chunk2 = 'd'
+chunk = f'{len(chunk):X}' + chunk2
+assert chunk == '3d'
+
+chunk = b'abc'
+chunk2 = 'd'
+chunk = f'{len(chunk):X}\r\n'.encode('ascii') + chunk \
+        + b'\r\n'
+assert chunk == b'3\r\nabc\r\n'
+
+# From 3.6.8 idlelib/pyshell.py
+# Bug was handling '''
+import os
+filename = '.'
+source = 'foo'
+source = (f"__file__ = r'''{os.path.abspath(filename)}'''\n"
+          + source + "\ndel __file__")
+print(source)

test/simple_source/bug36/03_build_tuple_call.py

@@ -0,0 +1,4 @@
+# From Python 3.6.5 email/message.py
+# Bug is in handling 'related' parameter
+def add_related(self, *args, **kw):
+    self._add_multipart('related', *args, _disp='inline', **kw)

test/simple_source/bug36/05_try_whiletrue.py

@@ -0,0 +1,19 @@
+# From 3.6 _collections.abc.py
+# Bug was try/execpt parsing detection since 3.6 removes
+# a JUMP_FORWARD from earlier 3.xs.
+# This could also get confused with try/else.
+
+# RUNNABLE!
+def iter(self):
+    i = 0
+    try:
+        while True:
+            v = self[i]
+            yield v
+            i += 1
+    except IndexError:
+        return
+
+
+A = [10, 20, 30]
+assert list(iter(A)) == A

test/simple_source/bug36/08_comp_gen_for.py

@@ -0,0 +1,19 @@
+# Bug in 3.3 weakset
+# Bug was not having a rule for 3.x "comp_for"
+
+# RUNNABLE!
+class WeakSet:
+    def __init__(self, data=None):
+        self.data = set(data)
+
+    def __iter__(self):
+        for item in self.data:
+            if item is not None:
+                yield item
+
+    def union(self, other):
+        return self.__class__(e for s in (self, other) for e in s)
+
+a = WeakSet([1, 2, 3])
+b = WeakSet([1, 3, 5])
+assert list(a.union(b)) == [1, 2, 3, 5]

test/simple_source/expression/04_subscript.py

@@ -0,0 +1,15 @@
+# From 3.6.8 idlelib/query.py
+# Bug was handling parenthesis around subscript in an assignment.
+
+# RUNNABLE!
+a = {'text': 1}
+b  = {'text': 3}
+for widget, entry, expect in (
+        (a, b, 1),
+        (None, b, 3)
+        ):
+    assert (widget or entry)['text'] == expect
+    (widget or entry)['text'] = 'A'
+
+assert a['text'] == 'A', "a[text] = %s != 'A'" % a['text']
+assert b['text'] == 'A', "a[text] = %s != 'A'" % b['text']

test/test_pyenvlib.py

@@ -36,8 +36,9 @@ python_versions = [v for v in magics.python_versions if
 # These include Jython, and Python bytecode changes pre release.
 
 TEST_VERSIONS = (
-               'pypy-2.4.0', 'pypy-2.6.1',
+               'pypy3-2.4.0', 'pypy-2.6.1',
                'pypy-5.0.1', 'pypy-5.3.1', 'pypy3.5-5.7.1-beta',
+               'pypy3.5-5.9.0', 'pypy3.5-6.0.0',
                'native') + tuple(python_versions)
 
 
@@ -57,19 +58,23 @@ test_options = {
     }
 
 for vers in TEST_VERSIONS:
-    if vers.startswith('pypy-'):
-        short_vers = vers[0:-2]
+    if vers.startswith('pypy'):
+        if vers.startswith('pypy3.'):
+            short_vers = vers[4:6]
+        else:
+            short_vers = vers[0:-2]
+
         test_options[vers] = (os.path.join(lib_prefix, vers, 'lib_pypy'),
                               PYC, 'python-lib'+short_vers)
-    if vers == 'native':
-        short_vers = os.path.basename(sys.path[-1])
-        test_options[vers] = (sys.path[-1],
-                              PYC, short_vers)
-
     else:
-        short_vers = vers[:3]
-        test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers),
-                              PYC, 'python-lib'+short_vers)
+        if vers == 'native':
+            short_vers = os.path.basename(sys.path[-1])
+            test_options[vers] = (sys.path[-1],
+                                  PYC, short_vers)
+        else:
+            short_vers = vers[:3]
+            test_options[vers] = (os.path.join(lib_prefix, vers, 'lib', 'python'+short_vers),
+                                  PYC, 'python-lib'+short_vers)
 
 def do_tests(src_dir, patterns, target_dir, start_with=None,
              do_verify=False, max_files=200):

test/test_pythonlib.py

@@ -79,7 +79,7 @@ for  vers in (2.7, 3.4, 3.5, 3.6):
 for  vers in (1.3, 1.4, 1.5,
               2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
               3.0, 3.1, 3.2, 3.3,
-              3.4, 3.5, 3.6, 3.7, 3.8, 'pypy3.2', 'pypy2.7'):
+              3.4, 3.5, 3.6, 3.7, 3.8, 'pypy3.2', 'pypy2.7', 'pypy3.6'):
     bytecode = "bytecode_%s" % vers
     key = "bytecode-%s" % vers
     test_options[key] =  (bytecode, PYC, bytecode, vers)

uncompyle6/.gitignore

@@ -0,0 +1 @@
+/.python-version

uncompyle6/parser.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2018 Rocky Bernstein
+#  Copyright (c) 2015-2019 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -587,14 +587,14 @@ class PythonParser(GenericASTBuilder):
         ##   designLists ::=
         ## Will need to redo semantic actiion
 
-        store        ::= STORE_FAST
-        store        ::= STORE_NAME
-        store        ::= STORE_GLOBAL
-        store        ::= STORE_DEREF
-        store        ::= expr STORE_ATTR
-        store        ::= store_subscr
-        store_subscr ::= expr expr STORE_SUBSCR
-        store        ::= unpack
+        store           ::= STORE_FAST
+        store           ::= STORE_NAME
+        store           ::= STORE_GLOBAL
+        store           ::= STORE_DEREF
+        store           ::= expr STORE_ATTR
+        store           ::= store_subscript
+        store_subscript ::= expr expr STORE_SUBSCR
+        store           ::= unpack
         '''
 
 

uncompyle6/parsers/parse2.py

@@ -96,10 +96,6 @@ class Python2Parser(PythonParser):
         for         ::= SETUP_LOOP expr for_iter store
                         for_block POP_BLOCK _come_froms
 
-        del_stmt ::= expr DELETE_SLICE+0
-        del_stmt ::= expr expr DELETE_SLICE+1
-        del_stmt ::= expr expr DELETE_SLICE+2
-        del_stmt ::= expr expr expr DELETE_SLICE+3
         del_stmt ::= delete_subscr
         delete_subscr ::= expr expr DELETE_SUBSCR
         del_stmt ::= expr DELETE_ATTR
@@ -372,6 +368,17 @@ class Python2Parser(PythonParser):
                 self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func)
                 custom_seen_ops.add(opname)
                 continue
+            elif opname.startswith('DELETE_SLICE'):
+                self.addRule("""
+                del_expr ::= expr
+                del_stmt ::= del_expr DELETE_SLICE+0
+                del_stmt ::= del_expr del_expr DELETE_SLICE+1
+                del_stmt ::= del_expr del_expr DELETE_SLICE+2
+                del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3
+                """, nop_func)
+                custom_seen_ops.add(opname)
+                self.check_reduce['del_expr'] = 'AST'
+                continue
             elif opname == 'DELETE_DEREF':
                 self.addRule("""
                    stmt           ::= del_deref_stmt
@@ -384,6 +391,7 @@ class Python2Parser(PythonParser):
                     del_stmt ::= delete_subscr
                     delete_subscr ::= expr expr DELETE_SUBSCR
                    """, nop_func)
+                self.check_reduce['delete_subscr'] = 'AST'
                 custom_seen_ops.add(opname)
                 continue
             elif opname == 'GET_ITER':
@@ -539,6 +547,10 @@ class Python2Parser(PythonParser):
         elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
             expr2 = ast[2]
             return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
+        elif lhs in ('delete_subscr', 'del_expr'):
+            op = ast[0][0]
+            return op.kind in ('and', 'or')
+
         return False
 
 class Python2ParserSingle(Python2Parser, PythonParserSingle):

uncompyle6/parsers/parse26.py

@@ -267,6 +267,8 @@ class Python26Parser(Python2Parser):
         # Note: preserve positions 0 2 and 4 for semantic actions
         conditional_not    ::= expr jmp_true  expr jf_cf_pop expr COME_FROM
         conditional        ::= expr jmp_false expr jf_cf_pop expr come_from_opt
+        conditional        ::= expr jmp_false expr ja_cf_pop expr
+
         expr               ::= conditional_not
 
         and                ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP

uncompyle6/parsers/parse27.py

@@ -41,7 +41,6 @@ class Python27Parser(Python2Parser):
         comp_body ::= set_comp_body
         comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
-        comp_iter     ::= comp_if
         comp_iter     ::= comp_body
 
         dict_comp_body ::= expr expr MAP_ADD
@@ -123,6 +122,7 @@ class Python27Parser(Python2Parser):
         conditional_true ::= expr JUMP_FORWARD expr COME_FROM
 
         conditional      ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM
+        conditional      ::= expr jmp_false expr JUMP_ABSOLUTE expr
         """
 
     def p_stmt27(self, args):

uncompyle6/parsers/parse3.py

@@ -99,8 +99,9 @@ class Python3Parser(PythonParser):
         sstmt ::= return RETURN_LAST
 
         return_if_stmts ::= return_if_stmt come_from_opt
-        return_if_stmts ::= _stmts return_if_stmt
-        return_if_stmt ::= ret_expr RETURN_END_IF
+        return_if_stmts ::= _stmts return_if_stmt _come_froms
+        return_if_stmt  ::= ret_expr RETURN_END_IF
+        returns         ::= _stmts return_if_stmt
 
         stmt      ::= break
         break     ::= BREAK_LOOP
@@ -953,11 +954,17 @@ class Python3Parser(PythonParser):
                                         opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
 
+                    else:
+                        rule = ('mklambda ::= %sLOAD_LAMBDA LOAD_CONST %s' %
+                                (('expr ' * stack_count), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
+
                     rule = ('mkfunc ::= %s%s%s%s' %
-                                ('expr ' * stack_count,
-                                 'load_closure ' * closure,
-                                 'LOAD_CONST ' * 2,
-                                 opname))
+                            ('expr ' * stack_count,
+                             'load_closure ' * closure,
+                             'LOAD_CONST ' * 2,
+                             opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
 
                     if has_get_iter_call_function1:
@@ -1149,7 +1156,9 @@ class Python3Parser(PythonParser):
         self.check_reduce['ifelsestmt'] = 'AST'
         self.check_reduce['annotate_tuple'] = 'noAST'
         self.check_reduce['kwarg'] = 'noAST'
-        self.check_reduce['try_except'] = 'AST'
+        if self.version < 3.6:
+            # 3.6+ can remove a JUMP_FORWARD which messes up our testing here
+            self.check_reduce['try_except'] = 'AST'
 
         # FIXME: remove parser errors caused by the below
         # self.check_reduce['while1elsestmt'] = 'noAST'
@@ -1206,7 +1215,7 @@ class Python3Parser(PythonParser):
             pass
         elif lhs == 'while1stmt':
 
-            # If there is a fall through to the COME_FROM_LOOP. then this is
+            # If there is a fall through to the COME_FROM_LOOP, then this is
             # not a while 1. So the instruction before should either be a
             # JUMP_BACK or the instruction before should not be the target of a
             # jump. (Well that last clause i not quite right; that target could be

uncompyle6/parsers/parse30.py

@@ -11,8 +11,8 @@ class Python30Parser(Python31Parser):
     def p_30(self, args):
         """
 
-        assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 POP_TOP
-        return_if_lambda  ::= RETURN_END_IF_LAMBDA POP_TOP
+        assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP
+        return_if_lambda  ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP
         compare_chained2  ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
 
         # FIXME: combine with parse3.2
@@ -27,10 +27,10 @@ class Python30Parser(Python31Parser):
         # instructions
 
         _ifstmts_jump  ::= c_stmts JUMP_FORWARD _come_froms POP_TOP COME_FROM
-        _ifstmts_jump  ::= c_stmts POP_TOP
+        _ifstmts_jump  ::= c_stmts COME_FROM POP_TOP
 
         # Used to keep index order the same in semantic actions
-        jb_pop_top     ::= JUMP_BACK POP_TOP
+        jb_pop_top     ::= JUMP_BACK _come_froms POP_TOP
 
         while1stmt     ::= SETUP_LOOP l_stmts COME_FROM_LOOP
         whileelsestmt  ::= SETUP_LOOP testexpr l_stmts
@@ -44,7 +44,7 @@ class Python30Parser(Python31Parser):
 
         ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel
         iflaststmtl ::= testexpr c_stmts_opt jb_pop_top
-        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE POP_TOP
+        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
 
         withasstmt    ::= expr setupwithas store suite_stmts_opt
                           POP_BLOCK LOAD_CONST COME_FROM_FINALLY
@@ -77,17 +77,18 @@ class Python30Parser(Python31Parser):
 
         # JUMP_IF_TRUE POP_TOP as a replacement
         comp_if       ::= expr jmp_false comp_iter
-        comp_if       ::= expr jmp_false comp_iter JUMP_BACK POP_TOP
-        comp_if_not   ::= expr jmp_true  comp_iter JUMP_BACK POP_TOP
+        comp_if       ::= expr jmp_false comp_iter JUMP_BACK COME_FROM POP_TOP
+        comp_if_not   ::= expr jmp_true  comp_iter JUMP_BACK COME_FROM POP_TOP
         comp_iter     ::= expr expr SET_ADD
         comp_iter     ::= expr expr LIST_APPEND
 
-        jump_forward_else     ::= JUMP_FORWARD POP_TOP
-        jump_absolute_else    ::= JUMP_ABSOLUTE POP_TOP
+        jump_forward_else     ::= JUMP_FORWARD COME_FROM POP_TOP
+        jump_absolute_else    ::= JUMP_ABSOLUTE COME_FROM POP_TOP
         except_suite          ::= c_stmts POP_EXCEPT jump_except POP_TOP
         except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY
-                                  _jump POP_TOP
-        jump_except           ::= JUMP_FORWARD POP_TOP
+                                  _jump COME_FROM POP_TOP
+        jump_except           ::= JUMP_FORWARD COME_FROM POP_TOP
+        jump_except           ::= JUMP_ABSOLUTE COME_FROM POP_TOP
         or                    ::= expr jmp_false expr jmp_true expr
         or                    ::= expr jmp_true expr
 
@@ -97,7 +98,7 @@ class Python30Parser(Python31Parser):
         # The below rules in fact are the same or similar.
 
         jmp_true       ::= JUMP_IF_TRUE POP_TOP
-        jmp_false      ::= JUMP_IF_FALSE POP_TOP
+        jmp_false      ::= JUMP_IF_FALSE _come_froms POP_TOP
 
         for_block      ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
 
@@ -106,10 +107,10 @@ class Python30Parser(Python31Parser):
         except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
                            POP_TOP END_FINALLY
 
-        return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
+        return_if_stmt ::= ret_expr RETURN_END_IF COME_FROM POP_TOP
         and            ::= expr jmp_false expr come_from_opt
         whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
-                           JUMP_BACK POP_TOP POP_BLOCK COME_FROM_LOOP
+                           JUMP_BACK COME_FROM POP_TOP POP_BLOCK COME_FROM_LOOP
         whilestmt      ::= SETUP_LOOP testexpr returns
                            POP_TOP POP_BLOCK COME_FROM_LOOP
 

uncompyle6/parsers/parse34.py

@@ -17,15 +17,10 @@ spark grammar differences over Python 3.3 for Python 3.4
 """
 
 from uncompyle6.parser import PythonParserSingle
-from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse33 import Python33Parser
 
 class Python34Parser(Python33Parser):
 
-    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
-        super(Python34Parser, self).__init__(debug_parser)
-        self.customized = {}
-
     def p_misc34(self, args):
         """
         expr ::= LOAD_ASSERT
@@ -34,6 +29,8 @@ class Python34Parser(Python33Parser):
         # passtmt is needed for semantic actions to add "pass"
         suite_stmts_opt ::= pass
 
+        whilestmt     ::= SETUP_LOOP testexpr returns come_froms POP_BLOCK COME_FROM_LOOP
+
         # Seems to be needed starting 3.4.4 or so
         while1stmt    ::= SETUP_LOOP l_stmts
                           COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP
@@ -45,7 +42,7 @@ class Python34Parser(Python33Parser):
         whileelsestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               else_suitel COME_FROM
 
-        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suitel
+        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK _come_froms POP_BLOCK else_suitel
                               COME_FROM_LOOP
 
         # Python 3.4+ optimizes the trailing two JUMPS away

uncompyle6/parsers/parse35.py

@@ -27,7 +27,6 @@ class Python35Parser(Python34Parser):
         while1stmt     ::= SETUP_LOOP l_stmts POP_BLOCK COME_FROM_LOOP
         while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK
                            POP_BLOCK else_suite COME_FROM_LOOP
-        whilestmt      ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM_LOOP
 
         # The following rule is for Python 3.5+ where we can have stuff like
         # while ..
@@ -105,7 +104,6 @@ class Python35Parser(Python34Parser):
         return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
         return_if_lambda   ::= RETURN_END_IF_LAMBDA COME_FROM
 
-
         jb_else     ::= JUMP_BACK ELSE
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
         ifelsestmtl ::= testexpr c_stmts_opt jb_else else_suitel
@@ -168,7 +166,7 @@ class Python35Parser(Python34Parser):
                    async_with_stmt ::= expr
                             BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
                             SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
-                            POP_BLOCK LOAD_CONST
+                            POP_BLOCK LOAD_CONST COME_FROM_ASYNC_WITH
                             WITH_CLEANUP_START
                             GET_AWAITABLE LOAD_CONST YIELD_FROM
                             WITH_CLEANUP_FINISH END_FINALLY
@@ -178,7 +176,7 @@ class Python35Parser(Python34Parser):
                    async_with_as_stmt ::= expr
                                BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
                                SETUP_ASYNC_WITH store suite_stmts_opt
-                               POP_BLOCK LOAD_CONST
+                               POP_BLOCK LOAD_CONST COME_FROM_ASYNC_WITH
                                WITH_CLEANUP_START
                                GET_AWAITABLE LOAD_CONST YIELD_FROM
                                WITH_CLEANUP_FINISH END_FINALLY

uncompyle6/parsers/parse36.py

@@ -40,6 +40,8 @@ class Python36Parser(Python35Parser):
 
         whilestmt       ::= SETUP_LOOP testexpr l_stmts_opt
                             JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
+        whilestmt       ::= SETUP_LOOP testexpr l_stmts_opt
+                            come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
 
         # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
         # RETURN_VALUE is meant. Specifcally this can happen in
@@ -53,6 +55,8 @@ class Python36Parser(Python35Parser):
         and  ::= expr jmp_false expr jmp_false
 
         jf_cf       ::= JUMP_FORWARD COME_FROM
+        cf_jf_else  ::= come_froms JUMP_FORWARD ELSE
+
         conditional ::= expr jmp_false expr jf_cf expr COME_FROM
 
         async_for_stmt     ::= SETUP_LOOP expr
@@ -102,6 +106,7 @@ class Python36Parser(Python35Parser):
 
         jb_cfs      ::= JUMP_BACK come_froms
         ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
+        ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel
 
         # In 3.6+, A sequence of statements ending in a RETURN can cause
         # JUMP_FORWARD END_FINALLY to be omitted from try middle
@@ -116,9 +121,12 @@ class Python36Parser(Python35Parser):
         try_except36     ::= SETUP_EXCEPT returns except_handler36
                              opt_come_from_except
         try_except36     ::= SETUP_EXCEPT suite_stmts
+        try_except36     ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                             except_handler36 opt_come_from_except
 
         # 3.6 omits END_FINALLY sometimes
         except_handler36 ::= COME_FROM_EXCEPT except_stmts
+        except_handler36 ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
         except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
 
         stmt             ::= tryfinally36
@@ -132,6 +140,8 @@ class Python36Parser(Python35Parser):
         stmt ::= tryfinally_return_stmt
         tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
                                    COME_FROM_FINALLY
+
+        compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
         """
 
     def customize_grammar_rules(self, tokens, customize):
@@ -162,16 +172,36 @@ class Python36Parser(Python35Parser):
                                   JUMP_ABSOLUTE END_FINALLY COME_FROM
                                   for_block pb_ja
                                   else_suite COME_FROM_LOOP
+
         """)
         self.check_reduce['call_kw'] = 'AST'
 
         for i, token in enumerate(tokens):
             opname = token.kind
 
-            if opname == 'FORMAT_VALUE':
+            if opname == 'LOAD_ASSERT':
+                if 'PyPy' in customize:
+                    rules_str = """
+                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
+            elif opname == 'FORMAT_VALUE':
                 rules_str = """
-                    expr ::= fstring_single
-                    fstring_single ::= expr FORMAT_VALUE
+                    expr            ::= fstring_single
+                    fstring_single  ::= expr FORMAT_VALUE
+                    expr            ::= fstring_expr
+                    fstring_expr    ::= expr FORMAT_VALUE
+
+                    str             ::= LOAD_CONST
+                    formatted_value ::= fstring_expr
+                    formatted_value ::= str
+
+                """
+                self.add_unique_doc_rules(rules_str, customize)
+            elif opname == 'FORMAT_VALUE_ATTR':
+                rules_str = """
+                expr            ::= fstring_single
+                fstring_single  ::= expr expr FORMAT_VALUE_ATTR
                 """
                 self.add_unique_doc_rules(rules_str, customize)
             elif opname == 'MAKE_FUNCTION_8':
@@ -219,12 +249,6 @@ class Python36Parser(Python35Parser):
                 v = token.attr
                 joined_str_n = "formatted_value_%s" % v
                 rules_str = """
-                    expr            ::= fstring_expr
-                    fstring_expr    ::= expr FORMAT_VALUE
-                    str             ::= LOAD_CONST
-                    formatted_value ::= fstring_expr
-                    formatted_value ::= str
-
                     expr                 ::= fstring_multi
                     fstring_multi        ::= joined_str BUILD_STRING
                     joined_str           ::= formatted_value+
@@ -232,6 +256,12 @@ class Python36Parser(Python35Parser):
                     %s                   ::= %sBUILD_STRING
                 """ % (joined_str_n, joined_str_n, "formatted_value " * v)
                 self.add_unique_doc_rules(rules_str, customize)
+                if 'FORMAT_VALUE_ATTR' in self.seen_ops:
+                    rules_str = """
+                      formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
+                      expr                 ::= formatted_value_attr
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
             elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
                 v = token.attr
                 rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)

uncompyle6/parsers/parse37.py

@@ -82,15 +82,31 @@ class Python37Parser(Python36Parser):
         call        ::= expr CALL_METHOD_0
 
         testtrue         ::= compare_chained37
+        testfalse        ::= compare_chained37_false
+
 
         compare_chained37   ::= expr compare_chained1a_37
         compare_chained37   ::= expr compare_chained1b_37
-        compare_chained2a_37 ::= expr COMPARE_OP POP_JUMP_IF_TRUE JUMP_FORWARD
-        compare_chained2b_37 ::= expr COMPARE_OP COME_FROM POP_JUMP_IF_FALSE JUMP_FORWARD ELSE
-        compare_chained1a_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                 compare_chained2a_37  ELSE POP_TOP COME_FROM
-        compare_chained1b_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                 compare_chained2b_37 POP_TOP JUMP_FORWARD COME_FROM
+        compare_chained37_false  ::= expr compare_chained1_false_37
+
+        compare_chained1a_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+        compare_chained1a_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained2a_37 ELSE POP_TOP COME_FROM
+        compare_chained1b_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained2b_37 POP_TOP JUMP_FORWARD COME_FROM
+
+        compare_chained1_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained2c_37 POP_TOP JUMP_FORWARD COME_FROM
+
+        compare_chained2a_37       ::= expr COMPARE_OP POP_JUMP_IF_TRUE JUMP_FORWARD
+        compare_chained2a_false_37 ::= expr COMPARE_OP POP_JUMP_IF_FALSE JUMP_FORWARD
+
+        compare_chained2b_37       ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD ELSE
+
+        compare_chained2c_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
+                                       compare_chained2a_false_37 ELSE
+
+        _ifstmts_jump        ::= c_stmts_opt come_froms
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/parse38.py

@@ -32,6 +32,7 @@ class Python38Parser(Python37Parser):
         stmt               ::= forelselaststmt38
         stmt               ::= forelselaststmtl38
         stmt               ::= tryfinally38
+        stmt               ::= try_elsestmtl38
         stmt               ::= try_except_ret38
         stmt               ::= try_except38
         stmt               ::= whilestmt38
@@ -72,7 +73,7 @@ class Python38Parser(Python37Parser):
                                SETUP_ASYNC_WITH POP_TOP
                                suite_stmts
                                POP_TOP POP_BLOCK
-                               BEGIN_FINALLY
+                               BEGIN_FINALLY COME_FROM_ASYNC_WITH
                                WITH_CLEANUP_START
                                GET_AWAITABLE LOAD_CONST YIELD_FROM
                                WITH_CLEANUP_FINISH END_FINALLY
@@ -81,7 +82,7 @@ class Python38Parser(Python37Parser):
                                SETUP_ASYNC_WITH store
                                suite_stmts
                                POP_TOP POP_BLOCK
-                               BEGIN_FINALLY
+                               BEGIN_FINALLY COME_FROM_ASYNC_WITH
                                WITH_CLEANUP_START
                                GET_AWAITABLE LOAD_CONST YIELD_FROM
                                WITH_CLEANUP_FINISH END_FINALLY
@@ -112,6 +113,10 @@ class Python38Parser(Python37Parser):
         except_cond1       ::= DUP_TOP expr COMPARE_OP jmp_false
                                POP_TOP POP_TOP POP_TOP
                                POP_EXCEPT
+
+        try_elsestmtl38    ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                               except_handler38 COME_FROM
+                               else_suitel opt_come_from_except
         try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
                                except_handler38
         try_except38       ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
@@ -119,8 +124,11 @@ class Python38Parser(Python37Parser):
         try_except_ret38   ::= SETUP_FINALLY expr POP_BLOCK
                                RETURN_VALUE except_ret38a
 
+        # Note: there is a suite_stmts_opt which seems
+        # to be bookkeeping which is not expressed in source code
         except_ret38       ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
-                               CALL_FINALLY RETURN_VALUE COME_FROM_FINALLY
+                               CALL_FINALLY RETURN_VALUE COME_FROM
+                               COME_FROM_FINALLY
                                suite_stmts_opt END_FINALLY
         except_ret38a      ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
                                expr ROT_FOUR
@@ -201,6 +209,10 @@ class Python38Parser(Python37Parser):
            forelsestmt        ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite
            forelselaststmt    ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec
            forelselaststmtl   ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel
+
+           tryelsestmtl3      ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
+                                  except_handler COME_FROM else_suitel
+                                  opt_come_from_except
            try_except         ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                                   except_handler opt_come_from_except
            tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK

uncompyle6/scanners/pypy35.py

@@ -1,6 +1,6 @@
-#  Copyright (c) 2017 by Rocky Bernstein
+#  Copyright (c) 2017, 2019 by Rocky Bernstein
 """
-Python PyPy 3.2 decompiler scanner.
+Python PyPy 3.5 decompiler scanner.
 
 Does some additional massaging of xdis-disassembled instructions to
 make things easier for decompilation.

uncompyle6/scanners/pypy36.py

@@ -0,0 +1,22 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+"""
+Python PyPy 3.6 decompiler scanner.
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+"""
+
+import uncompyle6.scanners.scanner36 as scan
+
+# bytecode verification, verify(), uses JUMP_OPS from here
+from xdis.opcodes import opcode_35 as opc  # is this right?
+JUMP_OPs = opc.JUMP_OPS
+
+# We base this off of 3.5
+class ScannerPyPy36(scan.Scanner36):
+    def __init__(self, show_asm):
+        # There are no differences in initialization between
+        # pypy 3.6 and 3.6
+        scan.Scanner36.__init__(self, show_asm, is_pypy=True)
+        self.version = 3.6
+        return

uncompyle6/scanners/scanner3.py

@@ -510,9 +510,9 @@ class Scanner3(Scanner):
                     next_offset = xdis.next_offset(op, self.opc, offset)
 
                 if label is None:
-                    if op in op3.hasjrel and op != self.opc.FOR_ITER:
+                    if op in self.opc.hasjrel and op != self.opc.FOR_ITER:
                         label = next_offset + oparg
-                    elif op in op3.hasjabs:
+                    elif op in self.opc.hasjabs:
                         if op in self.jump_if_pop:
                             if oparg > offset:
                                 label = oparg

uncompyle6/scanners/scanner36.py

@@ -17,23 +17,30 @@ JUMP_OPS = opc.JUMP_OPS
 
 class Scanner36(Scanner3):
 
-    def __init__(self, show_asm=None):
-        Scanner3.__init__(self, 3.6, show_asm)
+    def __init__(self, show_asm=None, is_pypy=False):
+        Scanner3.__init__(self, 3.6, show_asm, is_pypy)
         return
 
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
         tokens, customize = Scanner3.ingest(self, co, classname, code_objects, show_asm)
+        not_pypy36 = not (self.version == 3.6 and self.is_pypy)
         for t in tokens:
             # The lowest bit of flags indicates whether the
             # var-keyword argument is placed at the top of the stack
-            if t.op == self.opc.CALL_FUNCTION_EX and t.attr & 1:
+            if ( not_pypy36 and
+                 t.op == self.opc.CALL_FUNCTION_EX and t.attr & 1):
                 t.kind = 'CALL_FUNCTION_EX_KW'
                 pass
             elif t.op == self.opc.CALL_FUNCTION_KW:
                 t.kind = 'CALL_FUNCTION_KW_%s' % t.attr
-            elif t.op == self.opc.BUILD_MAP_UNPACK_WITH_CALL:
+            elif t.op == self.opc.FORMAT_VALUE:
+                if (t.attr & 0x4):
+                    t.kind = 'FORMAT_VALUE_ATTR'
+                    pass
+            elif ( not_pypy36 and
+                   t.op == self.opc.BUILD_MAP_UNPACK_WITH_CALL ):
                 t.kind = 'BUILD_MAP_UNPACK_WITH_CALL_%d' % t.attr
-            elif t.op == self.opc.BUILD_TUPLE_UNPACK_WITH_CALL:
+            elif ( not_pypy36 and t.op == self.opc.BUILD_TUPLE_UNPACK_WITH_CALL ):
                 t.kind = 'BUILD_TUPLE_UNPACK_WITH_CALL_%d' % t.attr
             pass
         return tokens, customize

uncompyle6/scanners/scanner37.py

@@ -22,13 +22,14 @@ This sets up opcodes Python's 3.7 and calls a generalized
 scanner routine for Python 3.
 """
 
+from uncompyle6.scanners.scanner36 import Scanner36
 from uncompyle6.scanners.scanner3 import Scanner3
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_37 as opc
 JUMP_OPs = opc.JUMP_OPS
 
-class Scanner37(Scanner3):
+class Scanner37(Scanner36):
 
     def __init__(self, show_asm=None):
         Scanner3.__init__(self, 3.7, show_asm)

uncompyle6/scanners/scanner38.py

@@ -22,15 +22,14 @@ This sets up opcodes Python's 3.8 and calls a generalized
 scanner routine for Python 3.
 """
 
-from __future__ import print_function
-
+from uncompyle6.scanners.scanner37 import Scanner37
 from uncompyle6.scanners.scanner3 import Scanner3
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_38 as opc
 JUMP_OPs = opc.JUMP_OPS
 
-class Scanner38(Scanner3):
+class Scanner38(Scanner37):
 
     def __init__(self, show_asm=None):
         Scanner3.__init__(self, 3.8, show_asm)

uncompyle6/semantics/consts.py

@@ -27,6 +27,77 @@ else:
     maxint = sys.maxint
 
 
+# Operator precidence
+# See https://docs.python.org/2/reference/expressions.html
+# or https://docs.python.org/3/reference/expressions.html
+# for a list.
+
+# Things at the top of this list below with low-value precidence will
+# tend to have parenthesis around them. Things at the bottom
+# of the list will tend not to have parenthesis around them.
+PRECEDENCE = {
+    'list':                   0,
+    'dict':                   0,
+    'unary_convert':          0,
+    'dict_comp':              0,
+    'set_comp':               0,
+    'set_comp_expr':          0,
+    'list_comp':              0,
+    'generator_exp':          0,
+
+    'attribute':              2,
+    'subscript':              2,
+    'subscript2':             2,
+    'store_subscript':        2,
+    'delete_subscr':          2,
+    'slice0':                 2,
+    'slice1':                 2,
+    'slice2':                 2,
+    'slice3':                 2,
+    'buildslice2':            2,
+    'buildslice3':            2,
+    'call':                   2,
+
+    'BINARY_POWER':           4,
+
+    'unary_expr':             6,
+
+    'BINARY_MULTIPLY':        8,
+    'BINARY_DIVIDE':          8,
+    'BINARY_TRUE_DIVIDE':     8,
+    'BINARY_FLOOR_DIVIDE':    8,
+    'BINARY_MODULO':          8,
+
+    'BINARY_ADD':             10,
+    'BINARY_SUBTRACT':        10,
+
+    'BINARY_LSHIFT':          12,
+    'BINARY_RSHIFT':          12,
+
+    'BINARY_AND':             14,
+    'BINARY_XOR':             16,
+    'BINARY_OR':              18,
+
+    'compare':                20,
+    'unary_not':              22,
+    'and':                    24,
+    'ret_and':                24,
+
+    'or':                     26,
+    'ret_or':                 26,
+
+    'conditional':            28,
+    'conditional_lamdba':     28,
+    'conditional_not_lamdba': 28,
+    'conditionalnot':         28,
+    'ret_cond':               28,
+
+    '_mklambda':              30,
+
+    'yield':                 101,
+    'yield_from':            101
+}
+
 LINE_LENGTH = 80
 
 # Some parse trees created below are used for comparing code
@@ -150,14 +221,17 @@ TABLE_DIRECT = {
     'DELETE_FAST':	        ( '%|del %{pattr}\n', ),
     'DELETE_NAME':	        ( '%|del %{pattr}\n', ),
     'DELETE_GLOBAL':	        ( '%|del %{pattr}\n', ),
-    'delete_subscr':            ( '%|del %c[%c]\n', 0, 1,),
-    'subscript':                ( '%c[%p]',
-                                      (0, 'expr'),
-                                      (1, 100) ),
-    'subscript2':               ( '%c[%c]',
-                                      (0, 'expr'),
+    'delete_subscr':            ( '%|del %p[%c]\n',
+                                  (0, 'expr', PRECEDENCE['subscript']), (1, 'expr') ),
+    'subscript':                ( '%p[%c]',
+                                      (0, 'expr', PRECEDENCE['subscript']),
                                       (1, 'expr') ),
-    'store_subscr':	        ( '%c[%c]', 0, 1),
+    'subscript2':               ( '%p[%c]',
+                                      (0, 'expr', PRECEDENCE['subscript']),
+                                      (1, 'expr') ),
+    'store_subscript':	        ( '%p[%c]',
+                                  (0, 'expr', PRECEDENCE['subscript']),
+                                  (1, 'expr') ),
     'STORE_FAST':	        ( '%{pattr}', ),
     'STORE_NAME':	        ( '%{pattr}', ),
     'STORE_GLOBAL':	        ( '%{pattr}', ),
@@ -179,16 +253,19 @@ TABLE_DIRECT = {
     'list_iter':	    ( '%c', 0 ),
     'list_for':		    ( ' for %c in %c%c', 2, 0, 3 ),
     'list_if':		    ( ' if %c%c', 0, 2 ),
-    'list_if_not':		( ' if not %p%c', (0, 22), 2 ),
+    'list_if_not':		( ' if not %p%c',
+                                  (0, 'expr', PRECEDENCE['unary_not']),
+                                  2 ),
     'lc_body':		    ( '', ),	# ignore when recursing
 
     'comp_iter':	    ( '%c', 0 ),
     'comp_if':		    ( ' if %c%c', 0, 2 ),
-    'comp_if_not':	    ( ' if not %p%c', (0, 22), 2 ),
+    'comp_if_not':	    ( ' if not %p%c',
+                              (0, 'expr', PRECEDENCE['unary_not']), 2 ),
     'comp_body':	    ( '', ),	# ignore when recusing
-    'set_comp_body':    ( '%c', 0 ),
-    'gen_comp_body':    ( '%c', 0 ),
-    'dict_comp_body':   ( '%c:%c', 1, 0 ),
+    'set_comp_body':        ( '%c', 0 ),
+    'gen_comp_body':        ( '%c', 0 ),
+    'dict_comp_body':       ( '%c:%c', 1, 0 ),
 
     'assign':		    ( '%|%c = %p\n', -1, (0, 200) ),
 
@@ -207,8 +284,10 @@ TABLE_DIRECT = {
     'conditional':      ( '%p if %p else %p', (2, 27), (0, 27), (4, 27) ),
     'conditional_true': ( '%p if 1 else %p', (0, 27), (2, 27) ),
     'ret_cond':         ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ),
-    'conditional_not':  ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27) ),
-    'ret_cond_not':     ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ),
+    'conditional_not':  ( '%p if not %p else %p',
+                          (2, 27),
+                          (0, "expr", PRECEDENCE['unary_not']),
+                          (4, 27) ),
     'conditional_lambda':
                         ( '%c if %c else %c',
                           (2, 'expr'), 0, 4 ),
@@ -256,7 +335,8 @@ TABLE_DIRECT = {
     'ifstmt':		    ( '%|if %c:\n%+%c%-', 0, 1 ),
     'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
     'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'testtrue':         ( 'not %p', (0, 22) ),
+    'testtrue':         ( 'not %p',
+                          (0, PRECEDENCE['unary_not']) ),
 
     'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
     'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
@@ -336,76 +416,6 @@ MAP = {
     'exprlist':		MAP_R0,
 }
 
-# Operator precidence
-# See https://docs.python.org/2/reference/expressions.html
-# or https://docs.python.org/3/reference/expressions.html
-# for a list.
-
-# Things at the top of this list below with low-value precidence will
-# tend to have parenthesis around them. Things at the bottom
-# of the list will tend not to have parenthesis around them.
-PRECEDENCE = {
-    'list':                   0,
-    'dict':                   0,
-    'unary_convert':          0,
-    'dict_comp':              0,
-    'set_comp':               0,
-    'set_comp_expr':          0,
-    'list_comp':              0,
-    'generator_exp':          0,
-
-    'attribute':              2,
-    'subscript':              2,
-    'subscript2':             2,
-    'slice0':                 2,
-    'slice1':                 2,
-    'slice2':                 2,
-    'slice3':                 2,
-    'buildslice2':            2,
-    'buildslice3':            2,
-    'call':                   2,
-
-    'BINARY_POWER':           4,
-
-    'unary_expr':             6,
-
-    'BINARY_MULTIPLY':        8,
-    'BINARY_DIVIDE':          8,
-    'BINARY_TRUE_DIVIDE':     8,
-    'BINARY_FLOOR_DIVIDE':    8,
-    'BINARY_MODULO':          8,
-
-    'BINARY_ADD':             10,
-    'BINARY_SUBTRACT':        10,
-
-    'BINARY_LSHIFT':          12,
-    'BINARY_RSHIFT':          12,
-
-    'BINARY_AND':             14,
-    'BINARY_XOR':             16,
-    'BINARY_OR':              18,
-
-    'compare':                20,
-    'unary_not':              22,
-    'and':                    24,
-    'ret_and':                24,
-
-    'or':                     26,
-    'ret_or':                 26,
-
-    'conditional':            28,
-    'conditional_lamdba':     28,
-    'conditional_not_lamdba': 28,
-    'conditionalnot':         28,
-    'ret_cond':               28,
-    'ret_cond_not':           28,
-
-    '_mklambda':              30,
-
-    'yield':                 101,
-    'yield_from':            101
-}
-
 ASSIGN_TUPLE_PARAM = lambda param_name: \
              SyntaxTree('expr', [ Token('LOAD_FAST', pattr=param_name) ])
 

uncompyle6/semantics/customize.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2018 by Rocky Bernstein
+#  Copyright (c) 2018-2019 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -48,13 +48,77 @@ def customize_for_version(self, is_pypy, version):
             'assign3':          ( '%|%c, %c, %c = %c, %c, %c\n',
                                   5, 6, 7, 0, 1, 2 ),
             })
-    if version < 3.0:
-        if version == 2.4:
-            def n_iftrue_stmt24(node):
-                self.template_engine(('%c', 0), node)
-                self.default(node)
-                self.prune()
-            self.n_iftrue_stmt24 = n_iftrue_stmt24
+    if  version >= 3.0:
+        TABLE_DIRECT.update({
+            # Gotta love Python for its futzing around with syntax like this
+            'raise_stmt2':	 ( '%|raise %c from %c\n', 0, 1),
+        })
+
+        if version >= 3.2:
+            TABLE_DIRECT.update({
+            'del_deref_stmt': ( '%|del %c\n', 0),
+            'DELETE_DEREF': ( '%{pattr}', 0 ),
+            })
+        from uncompyle6.semantics.customize3 import customize_for_version3
+        customize_for_version3(self, version)
+    else:  # < 3.0
+        if 2.4 <= version <= 2.6:
+            TABLE_DIRECT.update({
+                'comp_for':	( ' for %c in %c', 3, 1 ),
+            })
+        else:
+            TABLE_DIRECT.update({
+                'comp_for':	( ' for %c in %c%c', 2, 0, 3 ),
+            })
+
+        if version >= 2.5:
+            from uncompyle6.semantics.customize25 import customize_for_version25
+            customize_for_version25(self, version)
+
+            if version >= 2.6:
+                from uncompyle6.semantics.customize26_27 import customize_for_version26_27
+                customize_for_version26_27(self, version)
+                pass
+        else:  # < 2.5
+            global NAME_MODULE
+            NAME_MODULE = SyntaxTree('stmt',
+                              [ SyntaxTree('assign',
+                                    [ SyntaxTree('expr',
+                                          [Token('LOAD_GLOBAL', pattr='__name__',
+                                                 offset=0, has_arg=True)]),
+                                      SyntaxTree('store',
+                                          [ Token('STORE_NAME', pattr='__module__',
+                                                  offset=3, has_arg=True)])
+                                    ])])
+            TABLE_DIRECT.update({
+                'importmultiple': ( '%|import %c%c\n', 2, 3),
+                'import_cont'   : ( ', %c', 2),
+                'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-',
+                                    (1, 'suite_stmts_opt') ,
+                                    (5, 'suite_stmts_opt') )
+                })
+            if version == 2.4:
+                def n_iftrue_stmt24(node):
+                    self.template_engine(('%c', 0), node)
+                    self.default(node)
+                    self.prune()
+                self.n_iftrue_stmt24 = n_iftrue_stmt24
+            else:  # version <= 2.3:
+                TABLE_DIRECT.update({
+                    'if1_stmt':	( '%|if 1\n%+%c%-', 5 )
+                })
+                if version <= 2.1:
+                    TABLE_DIRECT.update({
+                        'importmultiple': ( '%c', 2 ),
+                        # FIXME: not quite right. We have indiividual imports
+                        # when there is in fact one: "import a, b, ..."
+                        'imports_cont': ( '%C%,', (1, 100, '\n') ),
+                    })
+                    pass
+                pass
+            pass # < 2.5
+
+        # < 3.0 continues
 
         TABLE_R.update({
             'STORE_SLICE+0':	( '%c[:]', 0 ),
@@ -87,109 +151,6 @@ def customize_for_version(self, is_pypy, version):
             self.prune() # stop recursing
         self.n_exec_smt = n_exec_stmt
 
-    else:
-        TABLE_DIRECT.update({
-            # Gotta love Python for its futzing around with syntax like this
-            'raise_stmt2':	 ( '%|raise %c from %c\n', 0, 1),
-        })
+        pass # < 3.0
 
-        if version >= 3.2:
-            TABLE_DIRECT.update({
-            'del_deref_stmt': ( '%|del %c\n', 0),
-            'DELETE_DEREF': ( '%{pattr}', 0 ),
-            })
-
-    if version <= 2.4:
-        TABLE_DIRECT.update({
-            'importmultiple': ( '%|import %c%c\n', 2, 3),
-            'import_cont'   : ( ', %c', 2),
-            'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-',
-                                (1, 'suite_stmts_opt') ,
-                                (5, 'suite_stmts_opt') )
-            })
-        if version == 2.3:
-            TABLE_DIRECT.update({
-                'if1_stmt':	( '%|if 1\n%+%c%-', 5 )
-            })
-
-        global NAME_MODULE
-        NAME_MODULE = SyntaxTree('stmt',
-                          [ SyntaxTree('assign',
-                                [ SyntaxTree('expr',
-                                      [Token('LOAD_GLOBAL', pattr='__name__',
-                                             offset=0, has_arg=True)]),
-                                  SyntaxTree('store',
-                                      [ Token('STORE_NAME', pattr='__module__',
-                                              offset=3, has_arg=True)])
-                                ])])
-        pass
-        if version <= 2.3:
-            if version <= 2.1:
-                TABLE_DIRECT.update({
-                    'importmultiple': ( '%c', 2 ),
-                    # FIXME: not quite right. We have indiividual imports
-                    # when there is in fact one: "import a, b, ..."
-                    'imports_cont': ( '%C%,', (1, 100, '\n') ),
-                })
-                pass
-            pass
-        pass
-    elif version >= 2.5:
-        ########################
-        # Import style for 2.5+
-        ########################
-        TABLE_DIRECT.update({
-            'importmultiple': ( '%|import %c%c\n', 2, 3 ),
-            'import_cont'   : ( ', %c', 2 ),
-            # With/as is allowed as "from future" thing in 2.5
-            # Note: It is safe to put the variables after "as" in parenthesis,
-            # and sometimes it is needed.
-            'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
-            'withasstmt':   ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
-        })
-
-        # In 2.5+ "except" handlers and the "finally" can appear in one
-        # "try" statement. So the below has the effect of combining the
-        # "tryfinally" with statement with the "try_except" statement
-        def tryfinallystmt(node):
-            if len(node[1][0]) == 1 and node[1][0][0] == 'stmt':
-                if node[1][0][0][0] == 'try_except':
-                    node[1][0][0][0].kind = 'tf_try_except'
-                if node[1][0][0][0] == 'tryelsestmt':
-                    node[1][0][0][0].kind = 'tf_tryelsestmt'
-            self.default(node)
-        self.n_tryfinallystmt = tryfinallystmt
-
-    ########################################
-    # Python 2.6+
-    #    except <condition> as <var>
-    # vs. older:
-    #    except <condition> , <var>
-    #
-    # For 2.6 we use the older syntax which
-    # matches how we parse this in bytecode
-    ########################################
-    if version > 2.6:
-        TABLE_DIRECT.update({
-            'except_cond2':	( '%|except %c as %c:\n', 1, 5 ),
-        })
-    else:
-        TABLE_DIRECT.update({
-            'except_cond3':	 ( '%|except %c, %c:\n', 1, 6 ),
-            'testtrue_then': ( 'not %p', (0, 22) ),
-
-        })
-
-    if 2.4 <= version <= 2.6:
-        TABLE_DIRECT.update({
-            'comp_for':	( ' for %c in %c', 3, 1 ),
-        })
-    else:
-        TABLE_DIRECT.update({
-            'comp_for':	( ' for %c in %c%c', 2, 0, 3 ),
-        })
-
-    if  version >= 3.0:
-        from uncompyle6.semantics.customize3 import customize_for_version3
-        customize_for_version3(self, version)
     return

uncompyle6/semantics/customize25.py

@@ -0,0 +1,50 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Isolate Python 2.5+ version-specific semantic actions here.
+"""
+
+from uncompyle6.semantics.consts import TABLE_DIRECT
+
+#######################
+# Python 2.5+ Changes #
+#######################
+def customize_for_version25(self, version):
+
+    ########################
+    # Import style for 2.5+
+    ########################
+    TABLE_DIRECT.update({
+        'except_cond3'  : ( '%|except %c, %c:\n',
+                            (1, 'expr'), (-2, 'store') ),
+        'importmultiple': ( '%|import %c%c\n', 2, 3 ),
+        'import_cont'   : ( ', %c', 2 ),
+        # With/as is allowed as "from future" thing in 2.5
+        # Note: It is safe to put the variables after "as" in parenthesis,
+        # and sometimes it is needed.
+        'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
+        'withasstmt':   ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
+    })
+
+    # In 2.5+ "except" handlers and the "finally" can appear in one
+    # "try" statement. So the below has the effect of combining the
+    # "tryfinally" with statement with the "try_except" statement
+    def tryfinallystmt(node):
+        if len(node[1][0]) == 1 and node[1][0][0] == 'stmt':
+            if node[1][0][0][0] == 'try_except':
+                node[1][0][0][0].kind = 'tf_try_except'
+            if node[1][0][0][0] == 'tryelsestmt':
+                node[1][0][0][0].kind = 'tf_tryelsestmt'
+        self.default(node)
+    self.n_tryfinallystmt = tryfinallystmt

uncompyle6/semantics/customize26_27.py

@@ -0,0 +1,39 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Isolate Python 2.6 and 2.7 version-specific semantic actions here.
+"""
+
+from uncompyle6.semantics.consts import TABLE_DIRECT
+
+def customize_for_version26_27(self, version):
+
+    ########################################
+    # Python 2.6+
+    #    except <condition> as <var>
+    # vs. older:
+    #    except <condition> , <var>
+    #
+    # For 2.6 we use the older syntax which
+    # matches how we parse this in bytecode
+    ########################################
+    if version > 2.6:
+        TABLE_DIRECT.update({
+            'except_cond2':	( '%|except %c as %c:\n', 1, 5 ),
+        })
+    else:
+        TABLE_DIRECT.update({
+            'testtrue_then': ( 'not %p', (0, 22) ),
+
+        })

uncompyle6/semantics/customize3.py

@@ -16,20 +16,28 @@
 """Isolate Python 3 version-specific semantic actions here.
 """
 
-from uncompyle6.semantics.consts import (
-    INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT, TABLE_R)
+from uncompyle6.semantics.consts import TABLE_DIRECT
 
 from xdis.code import iscode
-from xdis.util import COMPILER_FLAG_BIT
-from spark_parser.ast import GenericASTTraversalPruningException
-from uncompyle6.scanners.tok import Token
-from uncompyle6.semantics.helper import flatten_list
 from uncompyle6.semantics.make_function import make_function3_annotate
+from uncompyle6.semantics.customize35 import customize_for_version35
+from uncompyle6.semantics.customize36 import customize_for_version36
+from uncompyle6.semantics.customize37 import customize_for_version37
+from uncompyle6.semantics.customize38 import customize_for_version38
 
 def customize_for_version3(self, version):
     TABLE_DIRECT.update({
+        'comp_for'       : ( ' for %c in %c',
+                            (2, 'store') , (0, 'expr') ),
+        'conditionalnot' : ( '%c if not %c else %c',
+                            (2, 'expr') , (0, 'expr'), (4, 'expr') ),
+        'except_cond2'   : ( '%|except %c as %c:\n', 1, 5 ),
         'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
-        'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
+        'importmultiple' : ( '%|import %c%c\n', 2, 3 ),
+        'import_cont'    : ( ', %c', 2 ),
+        'store_locals'   : ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
+        'withstmt'       : ( '%|with %c:\n%+%c%-', 0, 3),
+        'withasstmt'     : ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
         })
 
     assert version >= 3.0
@@ -270,748 +278,23 @@ def customize_for_version3(self, version):
                                    (5, 'else_suitel') ),
         })
     if version >= 3.4:
-        ########################
-        # Python 3.4+ Additions
+        #######################
+        # Python 3.4+ Changes #
         #######################
         TABLE_DIRECT.update({
             'LOAD_CLASSDEREF':	( '%{pattr}', ),
             })
-
-        ########################
-        # Python 3.5+ Additions
-        #######################
         if version >= 3.5:
-            TABLE_DIRECT.update({
-                'await_expr':	       ( 'await %c', 0),
-                'await_stmt':	       ( '%|%c\n', 0),
-                'async_for_stmt':      (
-                    '%|async for %c in %c:\n%+%|%c%-\n\n', 9, 1, 25 ),
-                'async_forelse_stmt':  (
-                    '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
-                    9, 1, 25, (27, 'else_suite') ),
-                'async_with_stmt':     (
-                    '%|async with %c:\n%+%|%c%-',
-                    (0, 'expr'), 7 ),
-                'async_with_as_stmt':  (
-                    '%|async with %c as %c:\n%+%|%c%-',
-                    (0, 'expr'), (6, 'store'), 7),
-                'unmap_dict':	       ( '{**%C}', (0, -1, ', **') ),
-                # 'unmapexpr':	       ( '{**%c}', 0), # done by n_unmapexpr
-
-            })
-
-            def async_call(node):
-                self.f.write('async ')
-                node.kind == 'call'
-                p = self.prec
-                self.prec = 80
-                self.template_engine(('%c(%P)', 0, (1, -4, ', ',
-                                      100)), node)
-                self.prec = p
-                node.kind == 'async_call'
-                self.prune()
-            self.n_async_call = async_call
-            self.n_build_list_unpack = self.n_list
-
-            if version == 3.5:
-                def n_call(node):
-                    mapping = self._get_mapping(node)
-                    table = mapping[0]
-                    key = node
-                    for i in mapping[1:]:
-                        key = key[i]
-                        pass
-                    if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
-                        # Python 3.5 changes the stack position of
-                        # *args: kwargs come after *args whereas
-                        # in earlier Pythons, *args is at the end
-                        # which simplifies things from our
-                        # perspective.  Python 3.6+ replaces
-                        # CALL_FUNCTION_VAR_KW with
-                        # CALL_FUNCTION_EX We will just swap the
-                        # order to make it look like earlier
-                        # Python 3.
-                        entry = table[key.kind]
-                        kwarg_pos = entry[2][1]
-                        args_pos = kwarg_pos - 1
-                        # Put last node[args_pos] after subsequent kwargs
-                        while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
-                            # swap node[args_pos] with node[kwargs_pos]
-                            node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
-                            args_pos = kwarg_pos
-                            kwarg_pos += 1
-                    elif key.kind.startswith('CALL_FUNCTION_VAR'):
-                        # CALL_FUNCTION_VAR's top element of the stack contains
-                        # the variable argument list, then comes
-                        # annotation args, then keyword args.
-                        # In the most least-top-most stack entry, but position 1
-                        # in node order, the positional args.
-                        argc = node[-1].attr
-                        nargs = argc & 0xFF
-                        kwargs = (argc >> 8) & 0xFF
-                        # FIXME: handle annotation args
-                        if nargs > 0:
-                            template = ('%c(%C, ', 0, (1, nargs+1, ', '))
-                        else:
-                            template = ('%c(', 0)
-                        self.template_engine(template, node)
-
-                        args_node =  node[-2]
-                        if args_node in ('pos_arg', 'expr'):
-                            args_node = args_node[0]
-                        if args_node == 'build_list_unpack':
-                            template = ('*%P)', (0, len(args_node)-1, ', *', 100))
-                            self.template_engine(template, args_node)
-                        else:
-                            if len(node) - nargs > 3:
-                                template = ('*%c, %C)', nargs+1, (nargs+kwargs+1, -1, ', '))
-                            else:
-                                template = ('*%c)', nargs+1)
-                            self.template_engine(template, node)
-                        self.prune()
-
-                    self.default(node)
-                self.n_call = n_call
-
-            def n_function_def(node):
-                if self.version >= 3.6:
-                    code_node = node[0][0]
-                else:
-                    code_node = node[0][1]
-
-                is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
-                if (is_code and
-                    (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
-                    self.template_engine(('\n\n%|async def %c\n',
-                                          -2), node)
-                else:
-                    self.template_engine(('\n\n%|def %c\n', -2),
-                                         node)
-                self.prune()
-            self.n_function_def = n_function_def
-
-            def unmapexpr(node):
-                last_n = node[0][-1]
-                for n in node[0]:
-                    self.preorder(n)
-                    if n != last_n:
-                        self.f.write(', **')
-                        pass
-                    pass
-                self.prune()
-                pass
-            self.n_unmapexpr = unmapexpr
-
-            # FIXME: start here
-            def n_list_unpack(node):
-                """
-                prettyprint an unpacked list or tuple
-                """
-                p = self.prec
-                self.prec = 100
-                lastnode = node.pop()
-                lastnodetype = lastnode.kind
-
-                # If this build list is inside a CALL_FUNCTION_VAR,
-                # then the first * has already been printed.
-                # Until I have a better way to check for CALL_FUNCTION_VAR,
-                # will assume that if the text ends in *.
-                last_was_star = self.f.getvalue().endswith('*')
-
-                if lastnodetype.startswith('BUILD_LIST'):
-                    self.write('['); endchar = ']'
-                elif lastnodetype.startswith('BUILD_TUPLE'):
-                    # Tuples can appear places that can NOT
-                    # have parenthesis around them, like array
-                    # subscripts. We check for that by seeing
-                    # if a tuple item is some sort of slice.
-                    no_parens = False
-                    for n in node:
-                        if n == 'expr' and n[0].kind.startswith('build_slice'):
-                            no_parens = True
-                            break
-                        pass
-                    if no_parens:
-                        endchar = ''
-                    else:
-                        self.write('('); endchar = ')'
-                        pass
-
-                elif lastnodetype.startswith('BUILD_SET'):
-                    self.write('{'); endchar = '}'
-                elif lastnodetype.startswith('BUILD_MAP_UNPACK'):
-                    self.write('{*'); endchar = '}'
-                elif lastnodetype.startswith('ROT_TWO'):
-                    self.write('('); endchar = ')'
-                else:
-                    raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')
-
-                flat_elems = flatten_list(node)
-
-                self.indent_more(INDENT_PER_LEVEL)
-                sep = ''
-                for elem in flat_elems:
-                    if elem in ('ROT_THREE', 'EXTENDED_ARG'):
-                        continue
-                    assert elem == 'expr'
-                    line_number = self.line_number
-                    value = self.traverse(elem)
-                    if elem[0] == 'tuple':
-                        assert value[0] == '('
-                        assert value[-1] == ')'
-                        value = value[1:-1]
-                        if value[-1] == ',':
-                            # singleton tuple
-                            value = value[:-1]
-                    else:
-                        value = '*' + value
-                    if line_number != self.line_number:
-                        sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
-                    else:
-                        if sep != '': sep += ' '
-                    if not last_was_star:
-                        pass
-                    else:
-                        last_was_star = False
-                    self.write(sep, value)
-                    sep = ','
-                if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
-                    self.write(',')
-                self.write(endchar)
-                self.indent_less(INDENT_PER_LEVEL)
-
-                self.prec = p
-                self.prune()
-                return
-
-            self.n_tuple_unpack = n_list_unpack
-
-
-        if version >= 3.6:
-            ########################
-            # Python 3.6+ Additions
-            #######################
-
-            # Value 100 is important; it is exactly
-            # module/function precidence.
-            PRECEDENCE['call_kw']     = 100
-            PRECEDENCE['call_kw36']   = 100
-            PRECEDENCE['call_ex']     = 100
-            PRECEDENCE['call_ex_kw']  = 100
-            PRECEDENCE['call_ex_kw2'] = 100
-            PRECEDENCE['call_ex_kw3'] = 100
-            PRECEDENCE['call_ex_kw4'] = 100
-            PRECEDENCE['unmap_dict']  = 0
-
-            TABLE_DIRECT.update({
-                'tryfinally36':  ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
-                                   (1, 'returns'), 3 ),
-                'fstring_expr':   ( "{%c%{conversion}}", 0),
-                # FIXME: the below assumes the format strings
-                # don't have ''' in them. Fix this properly
-                'fstring_single': ( "f'''{%c%{conversion}}'''", 0),
-                'fstring_multi':  ( "f'''%c'''", 0),
-                'func_args36':    ( "%c(**", 0),
-                'try_except36':   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
-                'except_return':  ( '%|except:\n%+%c%-', 3 ),
-                'unpack_list':    ( '*%c', (0, 'list') ),
-                'tryfinally_return_stmt':
-                      ( '%|try:\n%+%c%-%|finally:\n%+%|return%-\n\n', 1 ),
-
-                'async_for_stmt36':  (
-                    '%|async for %c in %c:\n%+%c%-%-\n\n',
-                    (9, 'store'), (1, 'expr'), (18, 'for_block') ),
-
-                'call_ex' : (
-                    '%c(%p)',
-                    (0, 'expr'), (1, 100)),
-                'call_ex_kw' : (
-                    '%c(%p)',
-                    (0, 'expr'), (2, 100)),
-
-            })
-
-            TABLE_R.update({
-                'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)),
-                # Not quite right
-                'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')),
-                })
-
-            def build_unpack_tuple_with_call(node):
-
-                if node[0] == 'expr':
-                    tup = node[0][0]
-                else:
-                    tup = node[0]
-                    pass
-                assert tup == 'tuple'
-                self.call36_tuple(tup)
-
-                buwc = node[-1]
-                assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
-                for n in node[1:-1]:
-                    self.f.write(', *')
-                    self.preorder(n)
-                    pass
-                self.prune()
-                return
-            self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call
-
-            def build_unpack_map_with_call(node):
-                n = node[0]
-                if n == 'expr':
-                    n = n[0]
-                if n == 'dict':
-                    self.call36_dict(n)
-                    first = 1
-                    sep = ', **'
-                else:
-                    first = 0
-                    sep = '**'
-                for n in node[first:-1]:
-                    self.f.write(sep)
-                    self.preorder(n)
-                    sep = ', **'
-                    pass
-                self.prune()
-                return
-            self.n_build_map_unpack_with_call = build_unpack_map_with_call
-
-            def call_ex_kw2(node):
-                """Handle CALL_FUNCTION_EX 2  (have KW) but with
-                BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
-
-                # This is weird shit. Thanks Python!
-                self.preorder(node[0])
-                self.write('(')
-
-                assert node[1] == 'build_tuple_unpack_with_call'
-                btuwc = node[1]
-                tup = btuwc[0]
-                if tup == 'expr':
-                    tup = tup[0]
-                assert tup == 'tuple'
-                self.call36_tuple(tup)
-                assert node[2] == 'build_map_unpack_with_call'
-
-                self.write(', ')
-                d = node[2][0]
-                if d == 'expr':
-                    d = d[0]
-                assert d == 'dict'
-                self.call36_dict(d)
-
-                args = btuwc[1]
-                self.write(', *')
-                self.preorder(args)
-
-                self.write(', **')
-                star_star_args = node[2][1]
-                if star_star_args == 'expr':
-                    star_star_args = star_star_args[0]
-                self.preorder(star_star_args)
-                self.write(')')
-                self.prune()
-            self.n_call_ex_kw2 = call_ex_kw2
-
-            def call_ex_kw3(node):
-                """Handle CALL_FUNCTION_EX 1 (have KW) but without
-                BUILD_MAP_UNPACK_WITH_CALL"""
-                self.preorder(node[0])
-                self.write('(')
-                args = node[1][0]
-                if args == 'expr':
-                    args = args[0]
-                if args == 'tuple':
-                    if self.call36_tuple(args) > 0:
-                        self.write(', ')
-                        pass
-                    pass
-
-                self.write('*')
-                self.preorder(node[1][1])
-                self.write(', ')
-
-                kwargs = node[2]
-                if kwargs == 'expr':
-                    kwargs = kwargs[0]
-                if kwargs == 'dict':
-                    self.call36_dict(kwargs)
-                else:
-                    self.write('**')
-                    self.preorder(kwargs)
-                self.write(')')
-                self.prune()
-            self.n_call_ex_kw3 = call_ex_kw3
-
-            def call_ex_kw4(node):
-                """Handle CALL_FUNCTION_EX {1 or 2} but without
-                BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
-                self.preorder(node[0])
-                self.write('(')
-                args = node[1][0]
-                if args == 'tuple':
-                    if self.call36_tuple(args) > 0:
-                        self.write(', ')
-                        pass
-                    pass
-                else:
-                    self.write('*')
-                    self.preorder(args)
-                    self.write(', ')
-                    pass
-
-                kwargs = node[2]
-                if kwargs == 'expr':
-                    kwargs = kwargs[0]
-                call_function_ex = node[-1]
-                assert (call_function_ex == 'CALL_FUNCTION_EX_KW'
-                        or (self.version >= 3.6 and call_function_ex == 'CALL_FUNCTION_EX'))
-                # FIXME: decide if the below test be on kwargs == 'dict'
-                if (call_function_ex.attr & 1 and
-                    (not isinstance(kwargs, Token) and kwargs != 'attribute')
-                    and not kwargs[0].kind.startswith('kvlist')):
-                    self.call36_dict(kwargs)
-                else:
-                    self.write('**')
-                    self.preorder(kwargs)
-                self.write(')')
-                self.prune()
-            self.n_call_ex_kw4 = call_ex_kw4
-
-            def call36_tuple(node):
-                """
-                A tuple used in a call, these are like normal tuples but they
-                don't have the enclosing parenthesis.
-                """
-                assert node == 'tuple'
-                # Note: don't iterate over last element which is a
-                # BUILD_TUPLE...
-                flat_elems = flatten_list(node[:-1])
-
-                self.indent_more(INDENT_PER_LEVEL)
-                sep = ''
-
-                for elem in flat_elems:
-                    if elem in ('ROT_THREE', 'EXTENDED_ARG'):
-                        continue
-                    assert elem == 'expr'
-                    line_number = self.line_number
-                    value = self.traverse(elem)
-                    if line_number != self.line_number:
-                        sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
-                    self.write(sep, value)
-                    sep = ', '
-
-                self.indent_less(INDENT_PER_LEVEL)
-                return len(flat_elems)
-            self.call36_tuple = call36_tuple
-
-            def call36_dict(node):
-                """
-                A dict used in a call_ex_kw2, which are a dictionary items expressed
-                in a call. This should format to:
-                     a=1, b=2
-                In other words, no braces, no quotes around keys and ":" becomes
-                "=".
-
-                We will source-code use line breaks to guide us when to break.
-                """
-                p = self.prec
-                self.prec = 100
-
-                self.indent_more(INDENT_PER_LEVEL)
-                sep = INDENT_PER_LEVEL[:-1]
-                line_number = self.line_number
-
-                if  node[0].kind.startswith('kvlist'):
-                    # Python 3.5+ style key/value list in dict
-                    kv_node = node[0]
-                    l = list(kv_node)
-                    i = 0
-
-                    length = len(l)
-                    # FIXME: Parser-speed improved grammars will have BUILD_MAP
-                    # at the end. So in the future when everything is
-                    # complete, we can do an "assert" instead of "if".
-                    if kv_node[-1].kind.startswith("BUILD_MAP"):
-                        length -= 1
-
-                    # Respect line breaks from source
-                    while i < length:
-                        self.write(sep)
-                        name = self.traverse(l[i], indent='')
-                        # Strip off beginning and trailing quotes in name
-                        name = name[1:-1]
-                        if i > 0:
-                            line_number = self.indent_if_source_nl(line_number,
-                                                                   self.indent + INDENT_PER_LEVEL[:-1])
-                        line_number = self.line_number
-                        self.write(name, '=')
-                        value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
-                        self.write(value)
-                        sep = ", "
-                        if line_number != self.line_number:
-                            sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
-                            line_number = self.line_number
-                        i += 2
-                        pass
-                elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
-                    keys_node = node[-2]
-                    keys = keys_node.attr
-                    # from trepan.api import debug; debug()
-                    assert keys_node == 'LOAD_CONST' and isinstance(keys, tuple)
-                    for i in range(node[-1].attr):
-                        self.write(sep)
-                        self.write(keys[i], '=')
-                        value = self.traverse(node[i], indent='')
-                        self.write(value)
-                        sep = ", "
-                        if line_number != self.line_number:
-                            sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
-                            line_number = self.line_number
-                            pass
-                        pass
-                else:
-                    self.write("**")
-                    try:
-                        self.default(node)
-                    except GenericASTTraversalPruningException:
-                        pass
-
-                self.prec = p
-                self.indent_less(INDENT_PER_LEVEL)
-                return
-            self.call36_dict = call36_dict
-
-
-            FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
-
-            def n_except_suite_finalize(node):
-                if node[1] == 'returns' and self.hide_internal:
-                    # Process node[1] only.
-                    # The code after "returns", e.g. node[3], is dead code.
-                    # Adding it is wrong as it dedents and another
-                    # exception handler "except_stmt" afterwards.
-                    # Note it is also possible that the grammar is wrong here.
-                    # and this should not be "except_stmt".
-                    self.indent_more()
-                    self.preorder(node[1])
-                    self.indent_less()
-                else:
-                    self.default(node)
-                self.prune()
-            self.n_except_suite_finalize = n_except_suite_finalize
-
-            def n_formatted_value(node):
-                if node[0] == 'LOAD_CONST':
-                    self.write(node[0].attr)
-                    self.prune()
-                else:
-                    self.default(node)
-            self.n_formatted_value = n_formatted_value
-
-            def f_conversion(node):
-                node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '')
-
-            def fstring_expr(node):
-                f_conversion(node)
-                self.default(node)
-            self.n_fstring_expr = fstring_expr
-
-            def fstring_single(node):
-                f_conversion(node)
-                self.default(node)
-            self.n_fstring_single = fstring_single
-
-            # def kwargs_only_36(node):
-            #     keys = node[-1].attr
-            #     num_kwargs = len(keys)
-            #     values = node[:num_kwargs]
-            #     for i, (key, value) in enumerate(zip(keys, values)):
-            #         self.write(key + '=')
-            #         self.preorder(value)
-            #         if i < num_kwargs:
-            #             self.write(',')
-            #     self.prune()
-            #     return
-            # self.n_kwargs_only_36 = kwargs_only_36
-
-            def n_call_kw36(node):
-                self.template_engine(("%c(", 0), node)
-                keys = node[-2].attr
-                num_kwargs = len(keys)
-                num_posargs = len(node) - (num_kwargs + 2)
-                n = len(node)
-                assert n >= len(keys)+1, \
-                  'not enough parameters keyword-tuple values'
-                sep = ''
-
-                line_number = self.line_number
-                for i in range(1, num_posargs):
-                    self.write(sep)
-                    self.preorder(node[i])
-                    if line_number != self.line_number:
-                        sep = ",\n" + self.indent + "  "
-                    else:
-                        sep = ", "
-                    line_number = self.line_number
-
-                i = num_posargs
-                j = 0
-                # FIXME: adjust output for line breaks?
-                while i < n-2:
-                    self.write(sep)
-                    self.write(keys[j] + '=')
-                    self.preorder(node[i])
-                    if line_number != self.line_number:
-                        sep = ",\n" + self.indent + "  "
-                    else:
-                        sep = ", "
-                    i += 1
-                    j += 1
-                self.write(')')
-                self.prune()
-                return
-            self.n_call_kw36 = n_call_kw36
-
-            def starred(node):
-                l = len(node)
-                assert l > 0
-                pos_args = node[0]
-                if pos_args == 'expr':
-                    pos_args = pos_args[0]
-                if pos_args == 'tuple':
-                    build_tuple = pos_args[0]
-                    if build_tuple.kind.startswith('BUILD_TUPLE'):
-                        tuple_len = 0
-                    else:
-                        tuple_len = len(node) - 1
-                    star_start = 1
-                    template = '%C', (0, -1, ', ')
-                    self.template_engine(template, pos_args)
-                    if tuple_len == 0:
-                        self.write("*()")
-                        # That's it
-                        self.prune()
-                    self.write(', ')
-                else:
-                    star_start = 0
-                if l > 1:
-                    template = ( '*%C', (star_start, -1, ', *') )
-                else:
-                    template = ( '*%c', (star_start, 'expr') )
-
-                self.template_engine(template, node)
-                self.prune()
-
-            self.n_starred = starred
-
-            def return_closure(node):
-                # Nothing should be output here
-                self.prune()
-                return
-            self.n_return_closure = return_closure
-
-            if version >= 3.7:
-                ########################
-                # Python 3.7+ changes
-                #######################
-
-                PRECEDENCE['attribute37'] = 2
-                TABLE_DIRECT.update({
-                    'async_forelse_stmt':  (
-                        '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
-                        (7, 'store'), (1, 'expr'), (17, 'for_block'), (25, 'else_suite') ),
-                    'async_for_stmt':  (
-                        '%|async for %c in %c:\n%+%c%-%-\n\n',
-                        (7, 'store'), (1, 'expr'), (17, 'for_block')),
-                    'async_for_stmt37':  (
-                        '%|async for %c in %c:\n%+%c%-%-\n\n',
-                        (7, 'store'), (1, 'expr'), (16, 'for_block') ),
-                    'attribute37':  ( '%c.%[1]{pattr}', 0 ),
-                    'compare_chained1a_37': ( ' %[3]{pattr.replace("-", " ")} %p %p',
-                                             (0, 19),
-                                             (-4, 19)),
-                    'compare_chained1b_37': ( ' %[3]{pattr.replace("-", " ")} %p %p',
-                                             (0, 19),
-                                             (-4, 19)),
-                    'compare_chained2a_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)),
-
-                    'compare_chained2b_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)),
-
-                    })
-                if version >= 3.8:
-                    ########################
-                    # Python 3.8+ changes
-                    #######################
-
-                    # FIXME: pytest doesn't add proper keys in testing. Reinstate after we have fixed pytest.
-                    # for lhs in 'for forelsestmt forelselaststmt '
-                    #             'forelselaststmtl tryfinally38'.split():
-                    #     del TABLE_DIRECT[lhs]
-
-                    TABLE_DIRECT.update({
-                        'async_for_stmt38':  (
-                            '%|async for %c in %c:\n%+%c%-%-\n\n',
-                            (7, 'store'), (0, 'expr'), (8, 'for_block') ),
-
-                        'async_forelse_stmt38':  (
-                            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
-                            (7, 'store'), (0, 'expr'), (8, 'for_block'), (-1, 'else_suite') ),
-
-                        'async_with_stmt38': (
-                            '%|async with %c:\n%+%|%c%-',
-                            (0, 'expr'), 7),
-
-                        'async_with_as_stmt38':  (
-                            '%|async with %c as %c:\n%+%|%c%-',
-                            (0, 'expr'), (6, 'store'),
-                            (7, 'suite_stmts') ),
-
-                        'except_handler38a': (
-                            '%c', (-2, 'stmts') ),
-                        'except_ret38a': (
-                            'return %c', (4, 'expr') ),
-                        'except_ret38':  ( '%|return %c\n', (1, 'expr') ),
-                        'for38':            (
-                            '%|for %c in %c:\n%+%c%-\n\n',
-                            (2, 'store'),
-                            (0, 'expr'),
-                            (3, 'for_block') ),
-                        'forelsestmt38':    (
-                            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
-                            (2, 'store'),
-                            (0, 'expr'),
-                            (3, 'for_block'), -2 ),
-                        'forelselaststmt38': (
-                            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-',
-                            (2, 'store'),
-                            (0, 'expr'),
-                            (3, 'for_block'), -2 ),
-                        'forelselaststmtl38':	(
-                            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
-                            (2, 'store'),
-                            (0, 'expr'),
-                            (3, 'for_block'), -2 ),
-
-                        'whilestmt38': ( '%|while %c:\n%+%c%-\n\n',
-                                         (0, 'testexpr'), (1, 'l_stmts') ),
-                        'whileTruestmt38': ( '%|while True:\n%+%c%-\n\n',
-                                         (0, 'l_stmts') ),
-                        'tryfinally38': (
-                            '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
-                                   (3, 'returns'), 6 ),
-                        'try_except38': (
-                            '%|try:\n%+%c\n%-%|except:\n%|%-%c\n\n',
-                                   (-2, 'suite_stmts_opt'), (-1, 'except_handler38a') ),
-                        'try_except_ret38': (
-                            '%|try:\n%+%|return %c%-\n%|except:\n%+%|%c%-\n\n',
-                                   (1, 'expr'), (-1, 'except_ret38a') ),
-                    })
-                    pass  # version >= 3.8
-                pass
-
-            pass # version >= 3.6
-        pass # version >= 3.4
+            customize_for_version35(self, version)
+            if version >= 3.6:
+                customize_for_version36(self, version)
+                if version >= 3.7:
+                    customize_for_version37(self, version)
+                    if version >= 3.8:
+                        customize_for_version38(self, version)
+                        pass  # version >= 3.8
+                    pass # 3.7
+                pass # 3.6
+            pass # 3.5
+        pass # 3.4
     return

uncompyle6/semantics/customize35.py

@@ -0,0 +1,230 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Isolate Python 3.5 version-specific semantic actions here.
+"""
+
+from xdis.code import iscode
+from xdis.util import COMPILER_FLAG_BIT
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, TABLE_DIRECT)
+from uncompyle6.semantics.helper import flatten_list
+
+#######################
+# Python 3.5+ Changes #
+#######################
+def customize_for_version35(self, version):
+    TABLE_DIRECT.update({
+        'await_expr':	       ( 'await %c', 0),
+        'await_stmt':	       ( '%|%c\n', 0),
+        'async_for_stmt':      (
+            '%|async for %c in %c:\n%+%|%c%-\n\n', 9, 1, 25 ),
+        'async_forelse_stmt':  (
+            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+            9, 1, 25, (27, 'else_suite') ),
+        'async_with_stmt':     (
+            '%|async with %c:\n%+%|%c%-',
+            (0, 'expr'), 7 ),
+        'async_with_as_stmt':  (
+            '%|async with %c as %c:\n%+%|%c%-',
+            (0, 'expr'), (6, 'store'), 7),
+        'unmap_dict':	       ( '{**%C}', (0, -1, ', **') ),
+        # 'unmapexpr':	       ( '{**%c}', 0), # done by n_unmapexpr
+
+    })
+
+    def async_call(node):
+        self.f.write('async ')
+        node.kind == 'call'
+        p = self.prec
+        self.prec = 80
+        self.template_engine(('%c(%P)', 0, (1, -4, ', ',
+                              100)), node)
+        self.prec = p
+        node.kind == 'async_call'
+        self.prune()
+    self.n_async_call = async_call
+    self.n_build_list_unpack = self.n_list
+
+    def n_call(node):
+        mapping = self._get_mapping(node)
+        table = mapping[0]
+        key = node
+        for i in mapping[1:]:
+            key = key[i]
+            pass
+        if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
+            # Python 3.5 changes the stack position of
+            # *args: kwargs come after *args whereas
+            # in earlier Pythons, *args is at the end
+            # which simplifies things from our
+            # perspective.  Python 3.6+ replaces
+            # CALL_FUNCTION_VAR_KW with
+            # CALL_FUNCTION_EX We will just swap the
+            # order to make it look like earlier
+            # Python 3.
+            entry = table[key.kind]
+            kwarg_pos = entry[2][1]
+            args_pos = kwarg_pos - 1
+            # Put last node[args_pos] after subsequent kwargs
+            while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
+                # swap node[args_pos] with node[kwargs_pos]
+                node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
+                args_pos = kwarg_pos
+                kwarg_pos += 1
+        elif key.kind.startswith('CALL_FUNCTION_VAR'):
+            # CALL_FUNCTION_VAR's top element of the stack contains
+            # the variable argument list, then comes
+            # annotation args, then keyword args.
+            # In the most least-top-most stack entry, but position 1
+            # in node order, the positional args.
+            argc = node[-1].attr
+            nargs = argc & 0xFF
+            kwargs = (argc >> 8) & 0xFF
+            # FIXME: handle annotation args
+            if nargs > 0:
+                template = ('%c(%C, ', 0, (1, nargs+1, ', '))
+            else:
+                template = ('%c(', 0)
+            self.template_engine(template, node)
+
+            args_node =  node[-2]
+            if args_node in ('pos_arg', 'expr'):
+                args_node = args_node[0]
+            if args_node == 'build_list_unpack':
+                template = ('*%P)', (0, len(args_node)-1, ', *', 100))
+                self.template_engine(template, args_node)
+            else:
+                if len(node) - nargs > 3:
+                    template = ('*%c, %C)', nargs+1, (nargs+kwargs+1, -1, ', '))
+                else:
+                    template = ('*%c)', nargs+1)
+                self.template_engine(template, node)
+            self.prune()
+
+        self.default(node)
+    self.n_call = n_call
+
+    def n_function_def(node):
+        if self.version >= 3.6:
+            code_node = node[0][0]
+        else:
+            code_node = node[0][1]
+
+        is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
+        if (is_code and
+            (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
+            self.template_engine(('\n\n%|async def %c\n',
+                                  -2), node)
+        else:
+            self.template_engine(('\n\n%|def %c\n', -2),
+                                 node)
+        self.prune()
+    self.n_function_def = n_function_def
+
+    def unmapexpr(node):
+        last_n = node[0][-1]
+        for n in node[0]:
+            self.preorder(n)
+            if n != last_n:
+                self.f.write(', **')
+                pass
+            pass
+        self.prune()
+        pass
+    self.n_unmapexpr = unmapexpr
+
+    # FIXME: start here
+    def n_list_unpack(node):
+        """
+        prettyprint an unpacked list or tuple
+        """
+        p = self.prec
+        self.prec = 100
+        lastnode = node.pop()
+        lastnodetype = lastnode.kind
+
+        # If this build list is inside a CALL_FUNCTION_VAR,
+        # then the first * has already been printed.
+        # Until I have a better way to check for CALL_FUNCTION_VAR,
+        # will assume that if the text ends in *.
+        last_was_star = self.f.getvalue().endswith('*')
+
+        if lastnodetype.startswith('BUILD_LIST'):
+            self.write('['); endchar = ']'
+        elif lastnodetype.startswith('BUILD_TUPLE'):
+            # Tuples can appear places that can NOT
+            # have parenthesis around them, like array
+            # subscripts. We check for that by seeing
+            # if a tuple item is some sort of slice.
+            no_parens = False
+            for n in node:
+                if n == 'expr' and n[0].kind.startswith('build_slice'):
+                    no_parens = True
+                    break
+                pass
+            if no_parens:
+                endchar = ''
+            else:
+                self.write('('); endchar = ')'
+                pass
+
+        elif lastnodetype.startswith('BUILD_SET'):
+            self.write('{'); endchar = '}'
+        elif lastnodetype.startswith('BUILD_MAP_UNPACK'):
+            self.write('{*'); endchar = '}'
+        elif lastnodetype.startswith('ROT_TWO'):
+            self.write('('); endchar = ')'
+        else:
+            raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')
+
+        flat_elems = flatten_list(node)
+
+        self.indent_more(INDENT_PER_LEVEL)
+        sep = ''
+        for elem in flat_elems:
+            if elem in ('ROT_THREE', 'EXTENDED_ARG'):
+                continue
+            assert elem == 'expr'
+            line_number = self.line_number
+            value = self.traverse(elem)
+            if elem[0] == 'tuple':
+                assert value[0] == '('
+                assert value[-1] == ')'
+                value = value[1:-1]
+                if value[-1] == ',':
+                    # singleton tuple
+                    value = value[:-1]
+            else:
+                value = '*' + value
+            if line_number != self.line_number:
+                sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
+            else:
+                if sep != '': sep += ' '
+            if not last_was_star:
+                pass
+            else:
+                last_was_star = False
+            self.write(sep, value)
+            sep = ','
+        if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
+            self.write(',')
+        self.write(endchar)
+        self.indent_less(INDENT_PER_LEVEL)
+
+        self.prec = p
+        self.prune()
+        return
+
+    self.n_tuple_unpack = n_list_unpack

uncompyle6/semantics/customize36.py

@@ -0,0 +1,481 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Isolate Python 3.6 version-specific semantic actions here.
+"""
+
+from spark_parser.ast import GenericASTTraversalPruningException
+from uncompyle6.scanners.tok import Token
+from uncompyle6.semantics.helper import flatten_list
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT, TABLE_R)
+
+def escape_format(s):
+    return s.replace('\r', '\\r').\
+        replace('\n', '\\n').\
+        replace("'''", '"""')
+
+#######################
+# Python 3.6+ Changes #
+#######################
+
+def customize_for_version36(self, version):
+    # Value 100 is important; it is exactly
+    # module/function precidence.
+    PRECEDENCE['call_kw']     = 100
+    PRECEDENCE['call_kw36']   = 100
+    PRECEDENCE['call_ex']     = 100
+    PRECEDENCE['call_ex_kw']  = 100
+    PRECEDENCE['call_ex_kw2'] = 100
+    PRECEDENCE['call_ex_kw3'] = 100
+    PRECEDENCE['call_ex_kw4'] = 100
+    PRECEDENCE['unmap_dict']  = 0
+
+    TABLE_DIRECT.update({
+        'tryfinally36':     ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
+                              (1, 'returns'), 3 ),
+        'fstring_expr':     ( "{%c%{conversion}}",
+                              (0, 'expr') ),
+        # FIXME: the below assumes the format strings
+        # don't have ''' in them. Fix this properly
+        'fstring_single':   ( "f'''{%c%{conversion}}'''", 0),
+        'formatted_value_attr': ( "f'''{%c%{conversion}}%{string}'''",
+                                  (0, 'expr')),
+        'fstring_multi':    ( "f'''%c'''", 0),
+        'func_args36':      ( "%c(**", 0),
+        'try_except36':     ( '%|try:\n%+%c%-%c\n\n', 1, -2 ),
+        'except_return':    ( '%|except:\n%+%c%-', 3 ),
+        'unpack_list':      ( '*%c', (0, 'list') ),
+        'tryfinally_return_stmt':
+              ( '%|try:\n%+%c%-%|finally:\n%+%|return%-\n\n', 1 ),
+
+        'async_for_stmt36':  (
+            '%|async for %c in %c:\n%+%c%-%-\n\n',
+            (9, 'store'), (1, 'expr'), (18, 'for_block') ),
+
+        'call_ex' : (
+            '%c(%p)',
+            (0, 'expr'), (1, 100)),
+        'call_ex_kw' : (
+            '%c(%p)',
+            (0, 'expr'), (2, 100)),
+
+    })
+
+    TABLE_R.update({
+        'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)),
+        # Not quite right
+        'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')),
+        })
+
+    def build_unpack_tuple_with_call(node):
+
+        if node[0] == 'expr':
+            tup = node[0][0]
+        else:
+            tup = node[0]
+            pass
+        assert tup == 'tuple'
+        self.call36_tuple(tup)
+
+        buwc = node[-1]
+        assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
+        for n in node[1:-1]:
+            self.f.write(', *')
+            self.preorder(n)
+            pass
+        self.prune()
+        return
+    self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call
+
+    def build_unpack_map_with_call(node):
+        n = node[0]
+        if n == 'expr':
+            n = n[0]
+        if n == 'dict':
+            self.call36_dict(n)
+            first = 1
+            sep = ', **'
+        else:
+            first = 0
+            sep = '**'
+        for n in node[first:-1]:
+            self.f.write(sep)
+            self.preorder(n)
+            sep = ', **'
+            pass
+        self.prune()
+        return
+    self.n_build_map_unpack_with_call = build_unpack_map_with_call
+
+    def call_ex_kw2(node):
+        """Handle CALL_FUNCTION_EX 2  (have KW) but with
+        BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
+
+        # This is weird shit. Thanks Python!
+        self.preorder(node[0])
+        self.write('(')
+
+        assert node[1] == 'build_tuple_unpack_with_call'
+        btuwc = node[1]
+        tup = btuwc[0]
+        if tup == 'expr':
+            tup = tup[0]
+
+        if tup == 'LOAD_CONST':
+            self.write(', '.join(['"%s"' % t.replace('"','\\"') for t in tup.attr]))
+        else:
+            assert tup == 'tuple'
+            self.call36_tuple(tup)
+
+        assert node[2] == 'build_map_unpack_with_call'
+
+        self.write(', ')
+        d = node[2][0]
+        if d == 'expr':
+            d = d[0]
+        assert d == 'dict'
+        self.call36_dict(d)
+
+        args = btuwc[1]
+        self.write(', *')
+        self.preorder(args)
+
+        self.write(', **')
+        star_star_args = node[2][1]
+        if star_star_args == 'expr':
+            star_star_args = star_star_args[0]
+        self.preorder(star_star_args)
+        self.write(')')
+        self.prune()
+    self.n_call_ex_kw2 = call_ex_kw2
+
+    def call_ex_kw3(node):
+        """Handle CALL_FUNCTION_EX 1 (have KW) but without
+        BUILD_MAP_UNPACK_WITH_CALL"""
+        self.preorder(node[0])
+        self.write('(')
+        args = node[1][0]
+        if args == 'expr':
+            args = args[0]
+        if args == 'tuple':
+            if self.call36_tuple(args) > 0:
+                self.write(', ')
+                pass
+            pass
+
+        self.write('*')
+        self.preorder(node[1][1])
+        self.write(', ')
+
+        kwargs = node[2]
+        if kwargs == 'expr':
+            kwargs = kwargs[0]
+        if kwargs == 'dict':
+            self.call36_dict(kwargs)
+        else:
+            self.write('**')
+            self.preorder(kwargs)
+        self.write(')')
+        self.prune()
+    self.n_call_ex_kw3 = call_ex_kw3
+
+    def call_ex_kw4(node):
+        """Handle CALL_FUNCTION_EX {1 or 2} but without
+        BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
+        self.preorder(node[0])
+        self.write('(')
+        args = node[1][0]
+        if args == 'tuple':
+            if self.call36_tuple(args) > 0:
+                self.write(', ')
+                pass
+            pass
+        else:
+            self.write('*')
+            self.preorder(args)
+            self.write(', ')
+            pass
+
+        kwargs = node[2]
+        if kwargs == 'expr':
+            kwargs = kwargs[0]
+        call_function_ex = node[-1]
+        assert (call_function_ex == 'CALL_FUNCTION_EX_KW'
+                or (self.version >= 3.6 and call_function_ex == 'CALL_FUNCTION_EX'))
+        # FIXME: decide if the below test be on kwargs == 'dict'
+        if (call_function_ex.attr & 1 and
+            (not isinstance(kwargs, Token) and kwargs != 'attribute')
+            and not kwargs[0].kind.startswith('kvlist')):
+            self.call36_dict(kwargs)
+        else:
+            self.write('**')
+            self.preorder(kwargs)
+        self.write(')')
+        self.prune()
+    self.n_call_ex_kw4 = call_ex_kw4
+
+    def call36_tuple(node):
+        """
+        A tuple used in a call, these are like normal tuples but they
+        don't have the enclosing parenthesis.
+        """
+        assert node == 'tuple'
+        # Note: don't iterate over last element which is a
+        # BUILD_TUPLE...
+        flat_elems = flatten_list(node[:-1])
+
+        self.indent_more(INDENT_PER_LEVEL)
+        sep = ''
+
+        for elem in flat_elems:
+            if elem in ('ROT_THREE', 'EXTENDED_ARG'):
+                continue
+            assert elem == 'expr'
+            line_number = self.line_number
+            value = self.traverse(elem)
+            if line_number != self.line_number:
+                sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
+            self.write(sep, value)
+            sep = ', '
+
+        self.indent_less(INDENT_PER_LEVEL)
+        return len(flat_elems)
+    self.call36_tuple = call36_tuple
+
+    def call36_dict(node):
+        """
+        A dict used in a call_ex_kw2, which are a dictionary items expressed
+        in a call. This should format to:
+             a=1, b=2
+        In other words, no braces, no quotes around keys and ":" becomes
+        "=".
+
+        We will source-code use line breaks to guide us when to break.
+        """
+        p = self.prec
+        self.prec = 100
+
+        self.indent_more(INDENT_PER_LEVEL)
+        sep = INDENT_PER_LEVEL[:-1]
+        line_number = self.line_number
+
+        if  node[0].kind.startswith('kvlist'):
+            # Python 3.5+ style key/value list in dict
+            kv_node = node[0]
+            l = list(kv_node)
+            i = 0
+
+            length = len(l)
+            # FIXME: Parser-speed improved grammars will have BUILD_MAP
+            # at the end. So in the future when everything is
+            # complete, we can do an "assert" instead of "if".
+            if kv_node[-1].kind.startswith("BUILD_MAP"):
+                length -= 1
+
+            # Respect line breaks from source
+            while i < length:
+                self.write(sep)
+                name = self.traverse(l[i], indent='')
+                # Strip off beginning and trailing quotes in name
+                name = name[1:-1]
+                if i > 0:
+                    line_number = self.indent_if_source_nl(line_number,
+                                                           self.indent + INDENT_PER_LEVEL[:-1])
+                line_number = self.line_number
+                self.write(name, '=')
+                value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
+                self.write(value)
+                sep = ", "
+                if line_number != self.line_number:
+                    sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
+                    line_number = self.line_number
+                i += 2
+                pass
+        elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
+            keys_node = node[-2]
+            keys = keys_node.attr
+            # from trepan.api import debug; debug()
+            assert keys_node == 'LOAD_CONST' and isinstance(keys, tuple)
+            for i in range(node[-1].attr):
+                self.write(sep)
+                self.write(keys[i], '=')
+                value = self.traverse(node[i], indent='')
+                self.write(value)
+                sep = ", "
+                if line_number != self.line_number:
+                    sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
+                    line_number = self.line_number
+                    pass
+                pass
+        else:
+            self.write("**")
+            try:
+                self.default(node)
+            except GenericASTTraversalPruningException:
+                pass
+
+        self.prec = p
+        self.indent_less(INDENT_PER_LEVEL)
+        return
+    self.call36_dict = call36_dict
+
+
+    FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a', 'X':':X'}
+
+    def n_except_suite_finalize(node):
+        if node[1] == 'returns' and self.hide_internal:
+            # Process node[1] only.
+            # The code after "returns", e.g. node[3], is dead code.
+            # Adding it is wrong as it dedents and another
+            # exception handler "except_stmt" afterwards.
+            # Note it is also possible that the grammar is wrong here.
+            # and this should not be "except_stmt".
+            self.indent_more()
+            self.preorder(node[1])
+            self.indent_less()
+        else:
+            self.default(node)
+        self.prune()
+    self.n_except_suite_finalize = n_except_suite_finalize
+
+    def n_formatted_value(node):
+        if node[0] == 'LOAD_CONST':
+            value = node[0].attr
+            if isinstance(value, tuple):
+                self.write(node[0].attr)
+            else:
+                self.write(escape_format(node[0].attr))
+            self.prune()
+        else:
+            self.default(node)
+    self.n_formatted_value = n_formatted_value
+
+    def f_conversion(node):
+        fmt_node = node.data[1]
+        if fmt_node == 'expr' and fmt_node[0] == 'LOAD_CONST':
+            data = fmt_node[0].attr
+        else:
+            data = fmt_node.attr
+        node.conversion = FSTRING_CONVERSION_MAP.get(data, '')
+
+    def fstring_expr(node):
+        f_conversion(node)
+        self.default(node)
+    self.n_fstring_expr = fstring_expr
+
+    def fstring_single(node):
+        f_conversion(node)
+        self.default(node)
+    self.n_fstring_single = fstring_single
+
+    def formatted_value_attr(node):
+        f_conversion(node)
+        fmt_node = node.data[3]
+        if fmt_node == 'expr' and fmt_node[0] == 'LOAD_CONST':
+            node.string = escape_format(fmt_node[0].attr)
+        else:
+            node.string = fmt_node
+
+        self.default(node)
+    self.n_formatted_value_attr = formatted_value_attr
+
+    # def kwargs_only_36(node):
+    #     keys = node[-1].attr
+    #     num_kwargs = len(keys)
+    #     values = node[:num_kwargs]
+    #     for i, (key, value) in enumerate(zip(keys, values)):
+    #         self.write(key + '=')
+    #         self.preorder(value)
+    #         if i < num_kwargs:
+    #             self.write(',')
+    #     self.prune()
+    #     return
+    # self.n_kwargs_only_36 = kwargs_only_36
+
+    def n_call_kw36(node):
+        self.template_engine(("%c(", 0), node)
+        keys = node[-2].attr
+        num_kwargs = len(keys)
+        num_posargs = len(node) - (num_kwargs + 2)
+        n = len(node)
+        assert n >= len(keys)+1, \
+          'not enough parameters keyword-tuple values'
+        sep = ''
+
+        line_number = self.line_number
+        for i in range(1, num_posargs):
+            self.write(sep)
+            self.preorder(node[i])
+            if line_number != self.line_number:
+                sep = ",\n" + self.indent + "  "
+            else:
+                sep = ", "
+            line_number = self.line_number
+
+        i = num_posargs
+        j = 0
+        # FIXME: adjust output for line breaks?
+        while i < n-2:
+            self.write(sep)
+            self.write(keys[j] + '=')
+            self.preorder(node[i])
+            if line_number != self.line_number:
+                sep = ",\n" + self.indent + "  "
+            else:
+                sep = ", "
+            i += 1
+            j += 1
+        self.write(')')
+        self.prune()
+        return
+    self.n_call_kw36 = n_call_kw36
+
+    def starred(node):
+        l = len(node)
+        assert l > 0
+        pos_args = node[0]
+        if pos_args == 'expr':
+            pos_args = pos_args[0]
+        if pos_args == 'tuple':
+            build_tuple = pos_args[0]
+            if build_tuple.kind.startswith('BUILD_TUPLE'):
+                tuple_len = 0
+            else:
+                tuple_len = len(node) - 1
+            star_start = 1
+            template = '%C', (0, -1, ', ')
+            self.template_engine(template, pos_args)
+            if tuple_len == 0:
+                self.write("*()")
+                # That's it
+                self.prune()
+            self.write(', ')
+        else:
+            star_start = 0
+        if l > 1:
+            template = ( '*%C', (star_start, -1, ', *') )
+        else:
+            template = ( '*%c', (star_start, 'expr') )
+
+        self.template_engine(template, node)
+        self.prune()
+
+    self.n_starred = starred
+
+    def return_closure(node):
+        # Nothing should be output here
+        self.prune()
+        return
+    self.n_return_closure = return_closure

uncompyle6/semantics/customize37.py

@@ -0,0 +1,58 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Isolate Python 3.7 version-specific semantic actions here.
+"""
+
+from uncompyle6.semantics.consts import PRECEDENCE, TABLE_DIRECT
+
+def customize_for_version37(self, version):
+    ########################
+    # Python 3.7+ changes
+    #######################
+
+    PRECEDENCE['attribute37'] = 2
+    TABLE_DIRECT.update({
+        'async_forelse_stmt':  (
+            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+            (7, 'store'), (1, 'expr'), (17, 'for_block'), (25, 'else_suite') ),
+        'async_for_stmt':  (
+            '%|async for %c in %c:\n%+%c%-%-\n\n',
+            (7, 'store'), (1, 'expr'), (17, 'for_block')),
+        'async_for_stmt37':  (
+            '%|async for %c in %c:\n%+%c%-%-\n\n',
+            (7, 'store'), (1, 'expr'), (16, 'for_block') ),
+        'attribute37':  ( '%c.%[1]{pattr}', 0 ),
+        'compare_chained1a_37': (
+            ' %[3]{pattr.replace("-", " ")} %p %p',
+            (0, 19), (-4, 19)),
+        'compare_chained1_false_37': (
+            ' %[3]{pattr.replace("-", " ")} %p %p',
+            (0, 19), (-4, 19)),
+        'compare_chained1b_37': (
+            ' %[3]{pattr.replace("-", " ")} %p %p',
+            (0, 19), (-4, 19)),
+        'compare_chained2a_37': (
+            '%[1]{pattr.replace("-", " ")} %p',
+            (0, 19) ),
+        'compare_chained2b_37': (
+            '%[1]{pattr.replace("-", " ")} %p',
+            (0, 19) ),
+        'compare_chained2a_false_37': (
+            '%[1]{pattr.replace("-", " ")} %p',
+            (0, 19 ) ),
+        'compare_chained2c_37': (
+            '%[3]{pattr.replace("-", " ")} %p %p', (0, 19), (6, 19) ),
+
+        })

uncompyle6/semantics/customize38.py

@@ -0,0 +1,101 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""Isolate Python 3.6 version-specific semantic actions here.
+"""
+
+########################
+# Python 3.8+ changes
+#######################
+
+from uncompyle6.semantics.consts import TABLE_DIRECT
+
+def customize_for_version38(self, version):
+
+    # FIXME: pytest doesn't add proper keys in testing. Reinstate after we have fixed pytest.
+    # for lhs in 'for forelsestmt forelselaststmt '
+    #             'forelselaststmtl tryfinally38'.split():
+    #     del TABLE_DIRECT[lhs]
+
+    TABLE_DIRECT.update({
+        'async_for_stmt38':  (
+            '%|async for %c in %c:\n%+%c%-%-\n\n',
+            (7, 'store'), (0, 'expr'), (8, 'for_block') ),
+
+        'async_forelse_stmt38':  (
+            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+            (7, 'store'), (0, 'expr'), (8, 'for_block'), (-1, 'else_suite') ),
+
+        'async_with_stmt38': (
+            '%|async with %c:\n%+%|%c%-',
+            (0, 'expr'), 7),
+
+        'async_with_as_stmt38':  (
+            '%|async with %c as %c:\n%+%|%c%-',
+            (0, 'expr'), (6, 'store'),
+            (7, 'suite_stmts') ),
+
+        'except_handler38': (
+            '%c', (2, 'except_stmts') ),
+
+        'except_handler38a': (
+            '%c', (-2, 'stmts') ),
+
+        'except_ret38a': (
+            'return %c', (4, 'expr') ),
+
+        # Note: there is a suite_stmts_opt which seems
+        # to be bookkeeping which is not expressed in source code
+        'except_ret38':  ( '%|return %c\n', (1, 'expr') ),
+
+        'for38':            (
+            '%|for %c in %c:\n%+%c%-\n\n',
+            (2, 'store'),
+            (0, 'expr'),
+            (3, 'for_block') ),
+        'forelsestmt38':    (
+            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+            (2, 'store'),
+            (0, 'expr'),
+            (3, 'for_block'), -2 ),
+        'forelselaststmt38': (
+            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-',
+            (2, 'store'),
+            (0, 'expr'),
+            (3, 'for_block'), -2 ),
+        'forelselaststmtl38':	(
+            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+            (2, 'store'),
+            (0, 'expr'),
+            (3, 'for_block'), -2 ),
+
+        'whilestmt38': ( '%|while %c:\n%+%c%-\n\n',
+                         (0, 'testexpr'), (1, 'l_stmts') ),
+        'whileTruestmt38': ( '%|while True:\n%+%c%-\n\n',
+                         (0, 'l_stmts') ),
+        'try_elsestmtl38': (
+            '%|try:\n%+%c%-%c%|else:\n%+%c%-',
+            (1, 'suite_stmts_opt'),
+            (3, 'except_handler38'),
+            (5, 'else_suitel') ),
+        'try_except38': (
+            '%|try:\n%+%c\n%-%|except:\n%|%-%c\n\n',
+                   (-2, 'suite_stmts_opt'), (-1, 'except_handler38a') ),
+        'try_except_ret38': (
+            '%|try:\n%+%|return %c%-\n%|except:\n%+%|%c%-\n\n',
+                   (1, 'expr'), (-1, 'except_ret38a') ),
+        'tryfinally38': (
+            '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
+                   (3, 'returns'), 6 ),
+    })

uncompyle6/semantics/make_function.py

@@ -507,11 +507,47 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     args_node = node[-1]
 
+    annotate_dict = {}
+
     # Get a list of tree nodes that constitute the values for the "default
     # parameters"; these are default values that appear before any *, and are
     # not to be confused with keyword parameters which may appear after *.
-    if isinstance(args_node.attr, tuple):
-        pos_args, kw_args, annotate_argc  = args_node.attr
+    args_attr = args_node.attr
+
+    if isinstance(args_attr, tuple) or (self.version >= 3.6 and isinstance(args_attr, list)):
+        if len(args_attr) == 3:
+            pos_args, kw_args, annotate_argc  = args_attr
+        else:
+            pos_args, kw_args, annotate_argc, closure  = args_attr
+
+            i = -4
+            kw_pairs = 0
+            if closure:
+                # FIXME: fill in
+                i -= 1
+            if annotate_argc:
+                # Turn into subroutine and DRY with other use
+                annotate_node = node[i]
+                if annotate_node == 'expr':
+                    annotate_node = annotate_node[0]
+                    annotate_name_node = annotate_node[-1]
+                    if annotate_node == 'dict' and annotate_name_node.kind.startswith('BUILD_CONST_KEY_MAP'):
+                        types = [self.traverse(n, indent='') for n in annotate_node[:-2]]
+                        names = annotate_node[-2].attr
+                        l = len(types)
+                        assert l == len(names)
+                        for i in range(l): annotate_dict[names[i]] = types[i]
+                        pass
+                    pass
+                i -= 1
+            if kw_args:
+                kw_node = node[i]
+                if kw_node == 'expr':
+                    kw_node = kw_node[0]
+                if kw_node == 'dict':
+                    kw_pairs = kw_node[-1].attr
+
+
         # FIXME: there is probably a better way to classify this.
         have_kwargs = node[0].kind.startswith('kwarg') or node[0] == 'no_kwargs'
         if len(node) >= 4:
@@ -536,9 +572,26 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 default_values_start += 1
             defparams = node[default_values_start:default_values_start+args_node.attr[0]]
         else:
-            # args are first, before kwargs. Or there simply are no kwargs.
-            defparams = node[:args_node.attr[0]]
-            pass
+            if self.version < 3.6:
+                defparams = node[:args_node.attr[0]]
+                kw_args  = 0
+            else:
+                defparams = []
+                # FIXME: DRY with code below
+                default, kw_args, annotate_argc = args_node.attr[0:3]
+                if default:
+                    expr_node = node[0]
+                    if node[0] == 'pos_arg':
+                        expr_node = expr_node[0]
+                    assert expr_node == 'expr', "expecting mkfunc default node to be an expr"
+                    if (expr_node[0] == 'LOAD_CONST' and
+                        isinstance(expr_node[0].attr, tuple)):
+                        defparams = [repr(a) for a in expr_node[0].attr]
+                    elif expr_node[0] in frozenset(('list', 'tuple', 'dict', 'set')):
+                        defparams =  [self.traverse(n, indent='') for n in expr_node[0][:-1]]
+                else:
+                    defparams = []
+                pass
     else:
         if self.version < 3.6:
             defparams = node[:args_node.attr]
@@ -562,9 +615,22 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             kw_pairs = 0
             if closure:
                 # FIXME: fill in
+                annotate = node[i]
                 i -= 1
-            if annotate:
-                # FIXME: fill in
+            if annotate_argc:
+                # Turn into subroutine and DRY with other use
+                annotate_node = node[i]
+                if annotate_node == 'expr':
+                    annotate_node = annotate_node[0]
+                    annotate_name_node = annotate_node[-1]
+                    if annotate_node == 'dict' and annotate_name_node.kind.startswith('BUILD_CONST_KEY_MAP'):
+                        types = [self.traverse(n, indent='') for n in annotate_node[:-2]]
+                        names = annotate_node[-2].attr
+                        l = len(types)
+                        assert l == len(names)
+                        for i in range(l): annotate_dict[names[i]] = types[i]
+                        pass
+                    pass
                 i -= 1
             if kw_args:
                 kw_node = node[i]
@@ -654,6 +720,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             ast[-1] = ast_expr
             pass
     else:
+        # FIXME: add annotations here
         self.write("(", ", ".join(params))
     # self.println(indent, '#flags:\t', int(code.co_flags))
 
@@ -752,7 +819,10 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     if is_lambda:
         self.write(": ")
     else:
-        self.println("):")
+        self.write(')')
+        if annotate_dict and 'return' in annotate_dict:
+            self.write(' -> %s' % annotate_dict['return'])
+        self.println(":")
 
     if len(code.co_consts) > 0 and code.co_consts[0] is not None and not is_lambda: # ugly
         # docstring exists, dump it

uncompyle6/semantics/pysource.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2018 by Rocky Bernstein
+#  Copyright (c) 2015-2019 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -88,7 +88,8 @@ Python.
 #
 #     %p  like %c but sets the operator precedence.
 #         Its argument then is a tuple indicating the node
-#         index and the precidence value, an integer.
+#         index and the precedence value, an integer. If 3 items are given,
+#         the second item is the nonterminal name and the precedence is given last.
 #
 #     %C  evaluate children recursively, with sibling children separated by the
 #         given string.  It needs a 3-tuple: a starting node, the maximimum
@@ -616,7 +617,7 @@ class SourceWalker(GenericASTTraversal, object):
                 node[-2][0].kind = 'build_tuple2'
         self.default(node)
 
-    n_store_subscr = n_subscript = n_delete_subscr
+    n_store_subscript = n_subscript = n_delete_subscr
 
     # Note: this node is only in Python 2.x
     # FIXME: figure out how to get this into customization
@@ -811,6 +812,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.write(func_name)
 
         self.indent_more()
+
         self.make_function(node, is_lambda=False, code_node=code_node)
 
         if len(self.param_stack) > 1:
@@ -943,7 +945,7 @@ class SourceWalker(GenericASTTraversal, object):
         # FIXME: clean this up
         if self.version >= 3.0 and node == 'dict_comp':
             cn = node[1]
-        elif self.version < 2.7 and node == 'generator_exp':
+        elif self.version <= 2.7 and node == 'generator_exp':
             if node[0] == 'LOAD_GENEXPR':
                 cn = node[0]
             elif node[0] == 'load_closure':
@@ -1080,7 +1082,7 @@ class SourceWalker(GenericASTTraversal, object):
                     n = (k[3], k[1])
                     pass
                 elif k == 'comp_iter':
-                    n = k[1]
+                    n = k[0]
                     pass
                 pass
         else:
@@ -1812,9 +1814,6 @@ class SourceWalker(GenericASTTraversal, object):
             node[-2][0].kind = 'unpack_w_parens'
         self.default(node)
 
-    # except_cond3 is only in Python <= 2.6
-    n_except_cond3 = n_except_cond2
-
     def template_engine(self, entry, startnode):
         """The format template interpetation engine.  See the comment at the
         beginning of this module for the how we interpret format
@@ -1825,7 +1824,6 @@ class SourceWalker(GenericASTTraversal, object):
         # print(startnode)
         # print(entry[0])
         # print('======')
-
         fmt = entry[0]
         arg = 1
         i = 0
@@ -1838,7 +1836,11 @@ class SourceWalker(GenericASTTraversal, object):
             typ = m.group('type') or '{'
             node = startnode
             if m.group('child'):
-                node = node[int(m.group('child'))]
+                try:
+                    node = node[int(m.group('child'))]
+                except:
+                    from trepan.api import debug; debug()
+                    pass
 
             if   typ == '%':	self.write('%')
             elif typ == '+':
@@ -1872,7 +1874,18 @@ class SourceWalker(GenericASTTraversal, object):
                 arg += 1
             elif typ == 'p':
                 p = self.prec
-                (index, self.prec) = entry[arg]
+                tup = entry[arg]
+                assert isinstance(tup, tuple)
+                if len(tup) == 3:
+                    (index, nonterm_name, self.prec) = tup
+                    assert node[index] == nonterm_name, (
+                        "at %s[%d], expected '%s' node; got '%s'" % (
+                            node.kind, arg,  nonterm_name, node[index].kind)
+                        )
+                else:
+                    assert len(tup) == 2
+                    (index, self.prec) = entry[arg]
+
                 self.preorder(node[index])
                 self.prec = p
                 arg += 1

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.3.0'  # noqa
+VERSION='3.3.2'  # noqa