Merge branch 'master' into python-2.4

which are created via a call to create a subclass.

Should be more general though.

.gitignore

@@ -21,3 +21,4 @@
 ChangeLog
 __pycache__
 build
+nohup.out

Makefile

@@ -27,19 +27,20 @@ check:
 check-short: pytest
 	$(MAKE) -C test check-short
 
+# Note for 2.6 use <=3.0.1 see requirements-dev.txt
 #: Tests for Python 2.7, 3.3 and 3.4
-check-2.7 check-3.3 check-3.4: pytest
+check-2.6 check-2.7 check-3.3 check-3.4 check-3.5: pytest
 	$(MAKE) -C test $@
 
 #: Tests for Python 3.2 and 3.5 - pytest doesn't work here
 # Or rather 3.5 doesn't work not on Travis
-check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
+check-3.0 check-3.1 check-3.2 check-3.6:
 	$(MAKE) -C test $@
 
 check-3.7: pytest
 
-#:Tests for Python 2.6 (doesn't have pytest)
-check-2.4 check-2.5 check-2.6:
+#:Tests for Python 2.4-2.5 (don't have pytest)
+check-2.4 check-2.5:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0

NEWS

@@ -1,3 +1,11 @@
+uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
+
+- Python 3.x subclass and call parsing fixes
+- Allow/note running on Python 3.1
+- improve 3.5+ BUILD_MAP_UNPACK
+- DRY instruction building code between 2.x and 3.x
+- expand testing
+
 uncompyle6 3.1.1 2018-04-01 Easter April Fool's
 
 Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"

__pkginfo__.py

@@ -35,6 +35,8 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
+                'Programming Language :: Python :: 3.1',
+                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.4.8'
+export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'

pytest/test_fjt.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
-from xdis.bytecode import Bytecode
-from array import array
 def bug(state, slotstate):
     if state:
         if slotstate is not None:
@@ -25,14 +23,7 @@ def test_if_in_for():
     code = bug.func_code
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
-        scan.build_prev_op(n)
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
@@ -47,14 +38,7 @@ def test_if_in_for():
         #    {'start': 62, 'end': 63, 'type': 'for-else'}]
 
         code = bug_loop.__code__
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.build_prev_op(n)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
         assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
         assert scan.structs == [
@@ -68,14 +52,7 @@ def test_if_in_for():
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
     elif 3.2 < PYTHON_VERSION <= 3.4:
-        bytecode = Bytecode(code, scan.opc)
-        scan.code = array('B', code.co_code)
-        scan.lines = scan.build_lines_data(code)
-        scan.build_prev_op()
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
@@ -85,5 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        assert True, "FIXME: should note fixed"
+        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        assert True
     return

pytest/test_grammar.py

@@ -20,13 +20,17 @@ def test_grammar():
     # We have custom rules that create the below
     expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
-    unused_rhs = set(['list', 'mkfunc', 'dict',
+    unused_rhs = set(['list', 'mkfunc',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
-    expect_lhs.add('kvlist')
-    expect_lhs.add('kv3')
+
+    if PYTHON_VERSION > 2.6:
+        expect_lhs.add('kvlist')
+        expect_lhs.add('kv3')
+        unused_rhs.add('dict')
+
     if PYTHON3:
         expect_lhs.add('load_genexpr')
 
@@ -46,7 +50,7 @@ def test_grammar():
                 unused_rhs.add('call')
                 if PYTHON_VERSION == 3.5:
                     expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
                     pass
                 pass
             else:
@@ -85,6 +89,8 @@ def test_grammar():
             """.split())
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
+        if PYTHON_VERSION == 2.6:
+            opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
         ignore_set.add('LOAD_CLASSNAME')

pytest/test_single_compile.py

@@ -1,6 +1,6 @@
 from uncompyle6 import PYTHON_VERSION, deparse_code
 
-if PYTHON_VERSION >= 2.5:
+if PYTHON_VERSION >= 2.6:
     def test_single_mode():
         single_expressions = (
             'i = 1',

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest>=3.0.0
+pytest>=3.0.0,<=3.0.1
 flake8
-hypothesis<=3.8.3
+hypothesis<=3.0.0

test/Makefile

@@ -37,6 +37,7 @@ check-3.0: check-bytecode
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
@@ -201,6 +202,7 @@ check-bytecode-3.0:
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:

test/grammar-cover/grammar.sh

@@ -30,7 +30,7 @@ while [[ -n $1 ]]  ; do
         source ./admin-tools/setup-python-2.4.sh
     fi
     GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
-    pyenv local ${LONG_VERSION}
+    (cd ../.. && pyenv local ${LONG_VERSION})
     cd ./test
     if [[ -r $COVER_FILE ]]; then
 	rm $COVER_FILE

test/run-and-email.sh

@@ -14,7 +14,7 @@ function displaytime {
     printf '%d seconds\n' $S
 }
 
-PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.4.8 2.6.9"}
+PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.3.7 3.4.8 2.6.9 3.6.4"}
 # PYVERSION=${PYVERSION:-"3.5.5"}
 
 USER=${USER:-rocky}
@@ -28,6 +28,10 @@ for VERSION in $PYVERSION ; do
 
     if [[ $VERSION == '3.5.5' ]] ; then
 	MAX_TESTS=224
+    elif [[ $VERSION == '3.2.6' ]] ; then
+	MAX_TESTS=700
+    elif [[ $VERSION == '3.6.4' ]] ; then
+	MAX_TESTS=400
     else
 	MAX_TESTS=800
     fi
@@ -42,7 +46,7 @@ for VERSION in $PYVERSION ; do
       rc=$?
 
       echo Python Version $(pyenv local) >> $LOGFILE
-      echo "" >>LOGFILE
+      echo "" >>$LOGFILE
 
       typeset -i ALL_FILES_ENDTIME=$(date +%s)
       (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))

test/simple_source/bug31/05_abc_test.py

@@ -0,0 +1,18 @@
+# Extracted from Python 3.5 test_abc.py
+# Bug is class having only a single kwarg
+# subclass.
+import abc
+import unittest
+from inspect import isabstract
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract
+            def bar(self): pass  # concrete
+        assert C.__abstractmethods__, {"foo"}
+        assert isabstract(C)
+        pass
+
+test_abstractmethod_integration(None)

test/simple_source/bug33/02_named_and_kwargs.py

@@ -1,5 +1,6 @@
 # Python 3.6 subprocess.py bug
 # Bug is getting params correct: timeout before **kwargs
+import subprocess
 def call(*popenargs, timeout=None, **kwargs):
     return
 
@@ -14,6 +15,9 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
 # From 3.4 asyncio/locks.py
 # Bug was handling" "value=1, *"
 
+class Semaphore:
+    pass
+
 class BoundedSemaphore(Semaphore):
     def __init__(self, value=1, *, loop=None):
         super().__init__(value, loop=loop)

test/simple_source/bug35/04_call_function.py

@@ -1,5 +1,6 @@
 # From sql/schema.py and 3.5 _strptime.py
-# Note that kwargs comes before "positional" args
+# Bug was code not knowing which Python versions
+# have kwargs coming before positional args in code.
 
 # RUNNABLE!
 
@@ -58,3 +59,23 @@ def __call__(self, *args, **kwds):
 # From 3.6.4 shutil
 def unpack_archive(func, filename, dict, format_info, extract_dir=None):
     func(filename, extract_dir, **dict(format_info[2]))
+
+# From 3.5.5 test_xrdrlib.py
+import xdrlib
+def assertRaisesConversion(self, *args):
+   self.assertRaises(xdrlib.ConversionError, *args)
+
+# From 3.2.6 _pyio.py
+class BlockingIOError(IOError):
+    def __init__(self, errno, strerror, characters_written=5):
+        super().__init__(errno, strerror)
+
+# From urllib/parse.py
+# Bug was using a subclass made from a call (to namedtuple)
+from collections import namedtuple
+
+class ResultMixin(object):
+    pass
+
+class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin):
+    pass

test/simple_source/bug35/07_build_map_unpack.py

@@ -0,0 +1,10 @@
+# Adapted from Python 3.6 trace.py
+# Bug was in handling BUID_TUPLE_UNPACK created via
+# *opts.arguments
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('filename', nargs='?')
+parser.add_argument('arguments', nargs=argparse.REMAINDER)
+opts = parser.parse_args(["foo", "a", "b"])
+argv = opts.filename, *opts.arguments
+assert argv == ('foo', 'a', 'b')

test/stdlib/run-and-email.sh

@@ -2,7 +2,7 @@
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
 SUBJECT_PREFIX="stdlib unit testing for"
-for VERSION in 3.4.8 3.5.5 3.6.4 ; do
+for VERSION in 2.6.9 2.7.14 3.4.8 3.5.5 3.6.4 ; do
     typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
     if ! pyenv local $VERSION ; then

test/stdlib/runtests.sh

@@ -101,8 +101,10 @@ case $PYVERSION in
     3.5)
 	SKIP_TESTS=(
 	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_quopri.py]=1      # Fails in crontab environment?
 	)
 	;;
+
     3.6)
 	SKIP_TESTS=(
 	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation

uncompyle6/parsers/parse3.py

@@ -117,9 +117,14 @@ class Python3Parser(PythonParser):
 
         classdef ::= build_class store
 
+        # FIXME: we need to add these because don't detect this properly
+        # in custom rules. Specifically if one of the exprs is CALL_FUNCTION
+        # then we'll mistake that for the final CALL_FUNCTION.
+        # We can fix by triggering on the CALL_FUNCTION op
         # Python3 introduced LOAD_BUILD_CLASS
         # Other definitions are in a custom rule
         build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
 
         stmt ::= classdefdeco
         classdefdeco ::= classdefdeco1 store
@@ -424,7 +429,7 @@ class Python3Parser(PythonParser):
                         LOAD_CONST CALL_FUNCTION_n
         build_class ::= LOAD_BUILD_CLASS mkfunc
                         expr
-                        call_function
+                        call
                         CALL_FUNCTION_3
          '''
         # FIXME: I bet this can be simplified
@@ -627,7 +632,7 @@ class Python3Parser(PythonParser):
                 self.addRule(rule, nop_func)
                 rule = 'expr ::= build_list_unpack'
                 self.addRule(rule, nop_func)
-            elif opname_base == 'BUILD_MAP':
+            elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):
                 kvlist_n = "kvlist_%s" % token.attr
                 if opname == 'BUILD_MAP_n':
                     # PyPy sometimes has no count. Sigh.
@@ -646,23 +651,21 @@ class Python3Parser(PythonParser):
                         # FIXME: Use the attr
                         # so this doesn't run into exponential parsing time.
                         if opname.startswith('BUILD_MAP_UNPACK'):
-                            # FIXME: start here
-                            # rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
-                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
                             rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = 'dict ::= %s' % ('dict_entry ' * token.attr)
-                            self.addRule(rule, nop_func)
 
+                            # FIXME: start here. The LHS should be unmap_dict, not dict.
                             # FIXME: really we need a combination of dict_entry-like things.
                             # It just so happens the most common case is not to mix
                             # dictionary comphensions with dictionary, elements
                             if self.seen_LOAD_DICTCOMP:
                                 rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)
                                 self.addRule(rule, nop_func)
-
-                            rule = 'unmap_dict ::= %s%s' % (('dict ' * token.attr), opname)
+                            rule = """
+                             expr       ::= unmap_dict
+                             unmap_dict ::= %s%s
+                             """ % ('expr ' * token.attr, opname)
                         else:
                             rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
                             self.add_unique_rule(rule, opname, token.attr, customize)
@@ -680,7 +683,9 @@ class Python3Parser(PythonParser):
                 v = token.attr
                 rule = ('starred ::= %s %s' % ('expr ' * v, opname))
                 self.addRule(rule, nop_func)
-            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
+
+            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',
+                                 'BUILD_TUPLE_UNPACK'):
                 v = token.attr
 
                 is_LOAD_CLOSURE = False

uncompyle6/scanner.py

@@ -21,15 +21,22 @@ scanner/ingestion module. From here we call various version-specific
 scanners, e.g. for Python 2.7 or 3.4.
 """
 
+from array import array
 import sys
 
-from uncompyle6 import PYTHON3, IS_PYPY
+from uncompyle6 import PYTHON3, IS_PYPY, PYTHON_VERSION
 from uncompyle6.scanners.tok import Token
 import xdis
-from xdis.bytecode import instruction_size, extended_arg_val, next_offset
+from xdis.bytecode import (
+    Bytecode, instruction_size, extended_arg_val, next_offset)
 from xdis.magics import canonic_python_version
 from xdis.util import code2num
 
+if PYTHON_VERSION < 2.6:
+    from xdis.namedtuple24 import namedtuple
+else:
+    from collections import namedtuple
+
 # The byte code versions we support.
 # Note: these all have to be floats
 PYTHON_VERSIONS = frozenset((1.5,
@@ -88,11 +95,73 @@ class Scanner(object):
         # FIXME: This weird Python2 behavior is not Python3
         self.resetTokenClass()
 
-    def opname_for_offset(self, offset):
-        return self.opc.opname[self.code[offset]]
+    def build_instructions(self, co):
+        """
+        Create a list of instructions (a structured object rather than
+        an array of bytes) and store that in self.insts
+        """
+        # FIXME: remove this when all subsidiary functions have been removed.
+        # We should be able to get everything from the self.insts list.
+        self.code = array('B', co.co_code)
 
-    def op_name(self, op):
-        return self.opc.opname[op]
+        bytecode = Bytecode(co, self.opc)
+        self.build_prev_op()
+        self.insts = self.remove_extended_args(list(bytecode))
+        self.lines = self.build_lines_data(co)
+        self.offset2inst_index = {}
+        for i, inst in enumerate(self.insts):
+            self.offset2inst_index[inst.offset] = i
+
+        return bytecode
+
+    def build_lines_data(self, code_obj):
+        """
+        Generate various line-related helper data.
+        """
+
+        # Offset: lineno pairs, only for offsets which start line.
+        # Locally we use list for more convenient iteration using indices
+        linestarts = list(self.opc.findlinestarts(code_obj))
+        self.linestarts = dict(linestarts)
+
+        # 'List-map' which shows line number of current op and offset of
+        # first op on following line, given offset of op as index
+        lines = []
+        LineTuple = namedtuple('LineTuple', ['l_no', 'next'])
+
+        # Iterate through available linestarts, and fill
+        # the data for all code offsets encountered until
+        # last linestart offset
+        _, prev_line_no = linestarts[0]
+        offset = 0
+        for start_offset, line_no in linestarts[1:]:
+            while offset < start_offset:
+                lines.append(LineTuple(prev_line_no, start_offset))
+                offset += 1
+            prev_line_no = line_no
+
+        # Fill remaining offsets with reference to last line number
+        # and code length as start offset of following non-existing line
+        codelen = len(self.code)
+        while offset < codelen:
+            lines.append(LineTuple(prev_line_no, codelen))
+            offset += 1
+        return lines
+
+    def build_prev_op(self):
+        """
+        Compose 'list-map' which allows to jump to previous
+        op, given offset of current op as index.
+        """
+        code = self.code
+        codelen = len(code)
+        # 2.x uses prev 3.x uses prev_op. Sigh
+        # Until we get this sorted out.
+        self.prev = self.prev_op = [0]
+        for offset in self.op_range(0, codelen):
+            op = code[offset]
+            for _ in range(instruction_size(op, self.opc)):
+                self.prev_op.append(offset)
 
     def is_jump_forward(self, offset):
         """
@@ -330,6 +399,12 @@ class Scanner(object):
 
         return result
 
+    def opname_for_offset(self, offset):
+        return self.opc.opname[self.code[offset]]
+
+    def op_name(self, op):
+        return self.opc.opname[op]
+
     def op_range(self, start, end):
         """
         Iterate through positions of opcodes, skipping
@@ -339,11 +414,50 @@ class Scanner(object):
             yield start
             start += instruction_size(self.code[start], self.opc)
 
+    def remove_extended_args(self, instructions):
+        """Go through instructions removing extended ARG.
+        get_instruction_bytes previously adjusted the operand values
+        to account for these"""
+        new_instructions = []
+        last_was_extarg = False
+        n = len(instructions)
+        for i, inst in enumerate(instructions):
+            if (inst.opname == 'EXTENDED_ARG' and
+                i+1 < n and instructions[i+1].opname != 'MAKE_FUNCTION'):
+                last_was_extarg = True
+                starts_line = inst.starts_line
+                is_jump_target = inst.is_jump_target
+                offset = inst.offset
+                continue
+            if last_was_extarg:
+
+                # j = self.stmts.index(inst.offset)
+                # self.lines[j] = offset
+
+                new_inst= inst._replace(starts_line=starts_line,
+                                        is_jump_target=is_jump_target,
+                                        offset=offset)
+                inst = new_inst
+                if i < n:
+                    new_prev = self.prev_op[instructions[i].offset]
+                    j = instructions[i+1].offset
+                    old_prev = self.prev_op[j]
+                    while self.prev_op[j] == old_prev and j < n:
+                        self.prev_op[j] = new_prev
+                        j += 1
+
+            last_was_extarg = False
+            new_instructions.append(inst)
+        return new_instructions
+
     def remove_mid_line_ifs(self, ifs):
         """
         Go through passed offsets, filtering ifs
         located somewhere mid-line.
         """
+
+        # FIXME: this doesn't work for Python 3.6+
+
         filtered = []
         for i in ifs:
             # For each offset, if line number of current and next op
@@ -411,7 +525,7 @@ def get_scanner(version, is_pypy=False, show_asm=None):
 if __name__ == "__main__":
     import inspect, uncompyle6
     co = inspect.currentframe().f_code
-    scanner = get_scanner('2.7.13', True)
-    scanner = get_scanner(sys.version[:5], False)
+    # scanner = get_scanner('2.7.13', True)
+    # scanner = get_scanner(sys.version[:5], False)
     scanner = get_scanner(uncompyle6.PYTHON_VERSION, IS_PYPY, True)
-    tokens, customize = scanner.ingest(co, {})
+    tokens, customize = scanner.ingest(co, {}, show_asm='after')

uncompyle6/scanners/scanner2.py

@@ -35,21 +35,15 @@ Finally we save token information.
 
 from uncompyle6 import PYTHON_VERSION
 
-if PYTHON_VERSION < 2.6:
-    from xdis.namedtuple24 import namedtuple
-else:
-    from collections import namedtuple
-
-from array import array
 from copy import copy
 
 from xdis.code import iscode
 from xdis.bytecode import (
-    Bytecode, op_has_argument, instruction_size,
+    op_has_argument, instruction_size,
     _get_const_info)
 from xdis.util import code2num
 
-from uncompyle6.scanner import Scanner
+from uncompyle6.scanner import Scanner, Token
 
 class Scanner2(Scanner):
     def __init__(self, version, show_asm=None, is_pypy=False):
@@ -61,6 +55,57 @@ class Scanner2(Scanner):
         self.genexpr_name = '<genexpr>'
         self.load_asserts = set([])
 
+        # Create opcode classification sets
+        # Note: super initilization above initializes self.opc
+
+        # Ops that start SETUP_ ... We will COME_FROM with these names
+        # Some blocks and END_ statements. And they can start
+        # a new statement
+
+        self.statement_opcodes = frozenset([
+            self.opc.SETUP_LOOP,       self.opc.BREAK_LOOP,
+            self.opc.SETUP_FINALLY,    self.opc.END_FINALLY,
+            self.opc.SETUP_EXCEPT,     self.opc.POP_BLOCK,
+            self.opc.STORE_FAST,       self.opc.DELETE_FAST,
+            self.opc.STORE_DEREF,      self.opc.STORE_GLOBAL,
+            self.opc.DELETE_GLOBAL,    self.opc.STORE_NAME,
+            self.opc.DELETE_NAME,      self.opc.STORE_ATTR,
+            self.opc.DELETE_ATTR,      self.opc.STORE_SUBSCR,
+            self.opc.DELETE_SUBSCR,    self.opc.RETURN_VALUE,
+            self.opc.RAISE_VARARGS,    self.opc.POP_TOP,
+            self.opc.PRINT_EXPR,       self.opc.PRINT_ITEM,
+            self.opc.PRINT_NEWLINE,    self.opc.PRINT_ITEM_TO,
+            self.opc.PRINT_NEWLINE_TO, self.opc.CONTINUE_LOOP,
+            self.opc.JUMP_ABSOLUTE,    self.opc.EXEC_STMT,
+        ])
+
+        # Opcodes that can start a "store" non-terminal.
+        # FIXME: JUMP_ABSOLUTE is weird. What's up with that?
+        self.designator_ops = frozenset([
+            self.opc.STORE_FAST,    self.opc.STORE_NAME,
+            self.opc.STORE_GLOBAL,  self.opc.STORE_DEREF,   self.opc.STORE_ATTR,
+            self.opc.STORE_SLICE_0, self.opc.STORE_SLICE_1, self.opc.STORE_SLICE_2,
+            self.opc.STORE_SLICE_3, self.opc.STORE_SUBSCR,  self.opc.UNPACK_SEQUENCE,
+            self.opc.JUMP_ABSOLUTE
+        ])
+
+        # Python 2.7 has POP_JUMP_IF_{TRUE,FALSE}_OR_POP but < 2.7 doesn't
+        # Add an empty set make processing more uniform.
+        self.pop_jump_if_or_pop = frozenset([])
+
+        # opcodes with expect a variable number pushed values whose
+        # count is in the opcode. For parsing we generally change the
+        # opcode name to include that number.
+        self.varargs_ops = frozenset([
+            self.opc.BUILD_LIST,           self.opc.BUILD_TUPLE,
+            self.opc.BUILD_SLICE,          self.opc.UNPACK_SEQUENCE,
+            self.opc.MAKE_FUNCTION,        self.opc.CALL_FUNCTION,
+            self.opc.MAKE_CLOSURE,         self.opc.CALL_FUNCTION_VAR,
+            self.opc.CALL_FUNCTION_KW,     self.opc.CALL_FUNCTION_VAR_KW,
+            self.opc.DUP_TOPX,             self.opc.RAISE_VARARGS])
+
+
+
     @staticmethod
     def unmangle_name(name, classname):
         """Remove __ from the end of _name_ if it starts with __classname__
@@ -110,7 +155,8 @@ class Scanner2(Scanner):
         if not show_asm:
             show_asm = self.show_asm
 
-        bytecode = Bytecode(co, self.opc)
+        bytecode = self.build_instructions(co)
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             for instr in bytecode.get_instructions(co):
@@ -121,21 +167,10 @@ class Scanner2(Scanner):
 
         # "customize" is in the process of going away here
         customize = {}
-
         if self.is_pypy:
             customize['PyPy'] = 0
 
-        Token = self.Token # shortcut
-
-        codelen = self.setup_code(co)
-
-        self.build_lines_data(co, codelen)
-        self.build_prev_op(codelen)
-
-        self.insts = list(bytecode)
-        self.offset2inst_index = {}
-        for i, inst in enumerate(self.insts):
-            self.offset2inst_index[inst.offset] = i
+        codelen = len(self.code)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
@@ -146,8 +181,6 @@ class Scanner2(Scanner):
         self.load_asserts = set()
         for i in self.op_range(0, codelen):
 
-            self.offset2inst_index[inst.offset] = i
-
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -318,7 +351,7 @@ class Scanner2(Scanner):
                     if (offset in self.stmts and
                           self.code[offset+3] not in (self.opc.END_FINALLY,
                                                      self.opc.POP_BLOCK)):
-                        if ((offset in self.linestartoffsets and
+                        if ((offset in self.linestarts and
                             self.code[self.prev[offset]] == self.opc.JUMP_ABSOLUTE)
                             or self.code[target] == self.opc.FOR_ITER
                             or offset not in self.not_continue):
@@ -331,10 +364,7 @@ class Scanner2(Scanner):
                 if offset in self.return_end_ifs:
                     op_name = 'RETURN_END_IF'
 
-            if offset in self.linestartoffsets:
-                linestart = self.linestartoffsets[offset]
-            else:
-                linestart = None
+            linestart = self.linestarts.get(offset, None)
 
             if offset not in replace:
                 tokens.append(Token(
@@ -353,63 +383,6 @@ class Scanner2(Scanner):
             print()
         return tokens, customize
 
-    def setup_code(self, co):
-        """
-        Creates Python-independent bytecode structure (byte array) in
-        self.code and records previous instruction in self.prev
-        The size of self.code is returned
-        """
-        self.code = array('B', co.co_code)
-
-        n = -1
-        for i in self.op_range(0, len(self.code)):
-            if self.code[i] in (self.opc.RETURN_VALUE, self.opc.END_FINALLY):
-                n = i + 1
-                pass
-            pass
-        assert n > -1, "Didn't find RETURN_VALUE or END_FINALLY"
-        self.code = array('B', co.co_code[:n])
-
-        return n
-
-    def build_prev_op(self, n):
-        self.prev = [0]
-        # mapping addresses of instruction & argument
-        for i in self.op_range(0, n):
-            op = self.code[i]
-            self.prev.append(i)
-            if op_has_argument(op, self.opc):
-                self.prev.append(i)
-                self.prev.append(i)
-                pass
-            pass
-
-    def build_lines_data(self, co, n):
-        """
-        Initializes self.lines and self.linesstartoffsets
-        """
-        self.lines = []
-        linetuple = namedtuple('linetuple', ['l_no', 'next'])
-
-        # self.linestarts is a tuple of (offset, line number).
-        # Turn that in a has that we can index
-        self.linestarts = list(self.opc.findlinestarts(co))
-        self.linestartoffsets = {}
-        for offset, lineno in self.linestarts:
-            self.linestartoffsets[offset] = lineno
-
-        j = 0
-        (prev_start_byte, prev_line_no) = self.linestarts[0]
-        for (start_byte, line_no) in self.linestarts[1:]:
-            while j < start_byte:
-                self.lines.append(linetuple(prev_line_no, start_byte))
-                j += 1
-            prev_line_no = start_byte
-        while j < n:
-            self.lines.append(linetuple(prev_line_no, n))
-            j+=1
-        return
-
     def build_statement_indices(self):
         code = self.code
         start = 0
@@ -976,7 +949,8 @@ class Scanner2(Scanner):
                                          'end':   pre_rtarget})
 
                 # FIXME: this is yet another case were we need dominators.
-                if pre_rtarget not in self.linestartoffsets or self.version < 2.7:
+                if (pre_rtarget not in self.linestarts
+                    or self.version < 2.7):
                     self.not_continue.add(pre_rtarget)
 
                 if rtarget < end_offset:
@@ -1165,6 +1139,19 @@ class Scanner2(Scanner):
 
         return targets
 
+    def patch_continue(self, tokens, offset, op):
+        if op in (self.opc.JUMP_FORWARD, self.opc.JUMP_ABSOLUTE):
+            # FIXME: this is a hack to catch stuff like:
+            #   for ...
+            #     try: ...
+            #     except: continue
+            # the "continue" is not on a new line.
+            n = len(tokens)
+            if (n > 2 and
+                tokens[-1].kind == 'JUMP_BACK' and
+                self.code[offset+3] == self.opc.END_FINALLY):
+                tokens[-1].kind = intern('CONTINUE')
+
     # FIXME: combine with scanner3.py code and put into scanner.py
     def rem_or(self, start, end, instr, target=None, include_beyond_target=False):
         """
@@ -1204,3 +1191,17 @@ class Scanner2(Scanner):
             instr_offsets = filtered
             filtered = []
         return instr_offsets
+
+if __name__ == "__main__":
+    from uncompyle6 import PYTHON_VERSION
+    if 2.0 <= PYTHON_VERSION < 3.0:
+        import inspect
+        co = inspect.currentframe().f_code
+        from uncompyle6 import PYTHON_VERSION
+        tokens, customize = Scanner2(PYTHON_VERSION).ingest(co)
+        for t in tokens:
+            print(t)
+    else:
+        print("Need to be Python 2.x to demo; I am %s." %
+              PYTHON_VERSION)
+    pass

uncompyle6/scanners/scanner26.py

@@ -32,59 +32,21 @@ from uncompyle6.scanner import L65536
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_26
-from xdis.bytecode import Bytecode
 from xdis.bytecode import _get_const_info
 
+from uncompyle6.scanner import Token
+
 JUMP_OPS = opcode_26.JUMP_OPS
 
 class Scanner26(scan.Scanner2):
     def __init__(self, show_asm=False):
         super(Scanner26, self).__init__(2.6, show_asm)
-        self.statement_opcodes = frozenset([
-            self.opc.SETUP_LOOP,       self.opc.BREAK_LOOP,
-            self.opc.SETUP_FINALLY,    self.opc.END_FINALLY,
-            self.opc.SETUP_EXCEPT,     self.opc.POP_BLOCK,
-            self.opc.STORE_FAST,       self.opc.DELETE_FAST,
-            self.opc.STORE_DEREF,      self.opc.STORE_GLOBAL,
-            self.opc.DELETE_GLOBAL,    self.opc.STORE_NAME,
-            self.opc.DELETE_NAME,      self.opc.STORE_ATTR,
-            self.opc.DELETE_ATTR,      self.opc.STORE_SUBSCR,
-            self.opc.DELETE_SUBSCR,    self.opc.RETURN_VALUE,
-            self.opc.RAISE_VARARGS,    self.opc.POP_TOP,
-            self.opc.PRINT_EXPR,       self.opc.PRINT_ITEM,
-            self.opc.PRINT_NEWLINE,    self.opc.PRINT_ITEM_TO,
-            self.opc.PRINT_NEWLINE_TO, self.opc.CONTINUE_LOOP,
-            self.opc.JUMP_ABSOLUTE,    self.opc.EXEC_STMT,
-        ])
 
         # "setup" opcodes
         self.setup_ops = frozenset([
             self.opc.SETUP_EXCEPT, self.opc.SETUP_FINALLY,
             ])
 
-        # opcodes with expect a variable number pushed values whose
-        # count is in the opcode. For parsing we generally change the
-        # opcode name to include that number.
-        self.varargs_ops = frozenset([
-            self.opc.BUILD_LIST,           self.opc.BUILD_TUPLE,
-            self.opc.BUILD_SLICE,          self.opc.UNPACK_SEQUENCE,
-            self.opc.MAKE_FUNCTION,        self.opc.CALL_FUNCTION,
-            self.opc.MAKE_CLOSURE,         self.opc.CALL_FUNCTION_VAR,
-            self.opc.CALL_FUNCTION_KW,     self.opc.CALL_FUNCTION_VAR_KW,
-            self.opc.DUP_TOPX,             self.opc.RAISE_VARARGS])
-
-        # opcodes that store values into a variable
-        self.designator_ops = frozenset([
-            self.opc.STORE_FAST,    self.opc.STORE_NAME,
-            self.opc.STORE_GLOBAL,  self.opc.STORE_DEREF,   self.opc.STORE_ATTR,
-            self.opc.STORE_SLICE_0, self.opc.STORE_SLICE_1, self.opc.STORE_SLICE_2,
-            self.opc.STORE_SLICE_3, self.opc.STORE_SUBSCR,  self.opc.UNPACK_SEQUENCE,
-            self.opc.JUMP_ABSOLUTE
-        ])
-
-        # Python 2.7 has POP_JUMP_IF_{TRUE,FALSE}_OR_POP but < 2.7 doesn't
-        # Add an empty set make processing more uniform.
-        self.pop_jump_if_or_pop = frozenset([])
         return
 
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
@@ -106,7 +68,8 @@ class Scanner26(scan.Scanner2):
         if not show_asm:
             show_asm = self.show_asm
 
-        bytecode = Bytecode(co, self.opc)
+        bytecode = self.build_instructions(co)
+
         # show_asm = 'after'
         if show_asm in ('both', 'before'):
             for instr in bytecode.get_instructions(co):
@@ -119,17 +82,7 @@ class Scanner26(scan.Scanner2):
         if self.is_pypy:
             customize['PyPy'] = 1
 
-        Token = self.Token # shortcut
-
-        codelen = self.setup_code(co)
-
-        self.build_lines_data(co, codelen)
-        self.build_prev_op(codelen)
-
-        self.insts = list(bytecode)
-        self.offset2inst_index = {}
-        for i, inst in enumerate(self.insts):
-            self.offset2inst_index[inst.offset] = i
+        codelen = len(self.code)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
@@ -288,7 +241,7 @@ class Scanner26(scan.Scanner2):
                     if (offset in self.stmts
                         and self.code[offset+3] not in (self.opc.END_FINALLY,
                                                           self.opc.POP_BLOCK)):
-                        if ((offset in self.linestartoffsets and
+                        if ((offset in self.linestarts and
                             tokens[-1].kind == 'JUMP_BACK')
                             or offset not in self.not_continue):
                             op_name = 'CONTINUE'
@@ -309,10 +262,7 @@ class Scanner26(scan.Scanner2):
                 if offset in self.return_end_ifs:
                     op_name = 'RETURN_END_IF'
 
-            if offset in self.linestartoffsets:
-                linestart = self.linestartoffsets[offset]
-            else:
-                linestart = None
+            linestart = self.linestarts.get(offset, None)
 
             if offset not in replace:
                 tokens.append(Token(

uncompyle6/scanners/scanner27.py

@@ -23,28 +23,15 @@ class Scanner27(Scanner2):
         super(Scanner27, self).__init__(2.7, show_asm, is_pypy)
 
         # opcodes that start statements
-        self.statement_opcodes = frozenset([
-            self.opc.SETUP_LOOP,       self.opc.BREAK_LOOP,
-            self.opc.SETUP_FINALLY,    self.opc.END_FINALLY,
-            self.opc.SETUP_EXCEPT,
-            self.opc.POP_BLOCK,        self.opc.STORE_FAST, self.opc.DELETE_FAST,
-            self.opc.STORE_DEREF,      self.opc.STORE_GLOBAL,
-            self.opc.DELETE_GLOBAL,    self.opc.STORE_NAME,
-            self.opc.DELETE_NAME,      self.opc.STORE_ATTR,
-            self.opc.DELETE_ATTR,      self.opc.STORE_SUBSCR,
-            self.opc.DELETE_SUBSCR,    self.opc.RETURN_VALUE,
-            self.opc.RAISE_VARARGS,    self.opc.POP_TOP,
-            self.opc.PRINT_EXPR,       self.opc.PRINT_ITEM,
-            self.opc.PRINT_NEWLINE,    self.opc.PRINT_ITEM_TO,
-            self.opc.PRINT_NEWLINE_TO, self.opc.CONTINUE_LOOP,
-            self.opc.JUMP_ABSOLUTE,    self.opc.EXEC_STMT,
-            # New in 2.7
-            self.opc.SETUP_WITH,
-            self.opc.STORE_SLICE_0,    self.opc.STORE_SLICE_1,
-            self.opc.STORE_SLICE_2,    self.opc.STORE_SLICE_3,
-            self.opc.DELETE_SLICE_0,   self.opc.DELETE_SLICE_1,
-            self.opc.DELETE_SLICE_2,   self.opc.DELETE_SLICE_3,
-        ])
+        self.statement_opcodes = frozenset(
+            self.statement_opcodes | set([
+                # New in 2.7
+                self.opc.SETUP_WITH,
+                self.opc.STORE_SLICE_0,    self.opc.STORE_SLICE_1,
+                self.opc.STORE_SLICE_2,    self.opc.STORE_SLICE_3,
+                self.opc.DELETE_SLICE_0,   self.opc.DELETE_SLICE_1,
+                self.opc.DELETE_SLICE_2,   self.opc.DELETE_SLICE_3,
+            ]))
 
         # opcodes which expect a variable number pushed values and whose
         # count is in the opcode. For parsing we generally change the
@@ -83,19 +70,6 @@ class Scanner27(Scanner2):
 
         return
 
-    def patch_continue(self, tokens, offset, op):
-        if op in (self.opc.JUMP_FORWARD, self.opc.JUMP_ABSOLUTE):
-            # FIXME: this is a hack to catch stuff like:
-            #   for ...
-            #     try: ...
-            #     except: continue
-            # the "continue" is not on a new line.
-            n = len(tokens)
-            if (n > 2 and
-                tokens[-1].kind == 'JUMP_BACK' and
-                self.code[offset+3] == self.opc.END_FINALLY):
-                tokens[-1].kind = intern('CONTINUE')
-
     pass
 
 if __name__ == "__main__":

uncompyle6/scanners/scanner3.py

@@ -40,10 +40,8 @@ if PYTHON_VERSION < 2.6:
 else:
     from collections import namedtuple
 
-from array import array
-
 from xdis.code import iscode
-from xdis.bytecode import Bytecode, instruction_size, _get_const_info
+from xdis.bytecode import instruction_size, _get_const_info
 
 from uncompyle6.scanner import Token, parse_fn_counts
 import xdis
@@ -104,7 +102,7 @@ class Scanner3(Scanner):
 
         self.statement_opcodes = frozenset(statement_opcodes) | self.setup_ops_no_loop
 
-        # Opcodes that can start a designator non-terminal.
+        # Opcodes that can start a "store" non-terminal.
         # FIXME: JUMP_ABSOLUTE is weird. What's up with that?
         self.designator_ops = frozenset([
             self.opc.STORE_FAST,    self.opc.STORE_NAME, self.opc.STORE_GLOBAL,
@@ -138,6 +136,7 @@ class Scanner3(Scanner):
                 (self.opc.JUMP_FORWARD,),
                 (self.opc.JUMP_ABSOLUTE,)]
 
+        # FIXME: remove this and use instead info from xdis.
         # Opcodes that take a variable number of arguments
         # (expr's)
         varargs_ops = set([
@@ -148,52 +147,21 @@ class Scanner3(Scanner):
 
         if is_pypy:
             varargs_ops.add(self.opc.CALL_METHOD)
-        if self.version >= 3.6:
-            varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
-            # Below is in bit order, "default = bit 0, closure = bit 3
-            self.MAKE_FUNCTION_FLAGS = tuple("""
-             default keyword-only annotation closure""".split())
+        if self.version >= 3.5:
+            varargs_ops |= set([self.opc.BUILD_SET_UNPACK,
+                                self.opc.BUILD_MAP_UNPACK,   # we will handle this later
+                                self.opc.BUILD_LIST_UNPACK,
+                                self.opc.BUILD_TUPLE_UNPACK])
+            if self.version >= 3.6:
+                varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
+                # Below is in bit order, "default = bit 0, closure = bit 3
+                self.MAKE_FUNCTION_FLAGS = tuple("""
+                 default keyword-only annotation closure""".split())
 
         self.varargs_ops = frozenset(varargs_ops)
         # FIXME: remove the above in favor of:
         # self.varargs_ops = frozenset(self.opc.hasvargs)
 
-    def remove_extended_args(self, instructions):
-        """Go through instructions removing extended ARG.
-        get_instruction_bytes previously adjusted the operand values
-        to account for these"""
-        new_instructions = []
-        last_was_extarg = False
-        n = len(instructions)
-        for i, inst in enumerate(instructions):
-            if (inst.opname == 'EXTENDED_ARG' and
-                i+1 < n and instructions[i+1].opname != 'MAKE_FUNCTION'):
-                last_was_extarg = True
-                starts_line = inst.starts_line
-                is_jump_target = inst.is_jump_target
-                offset = inst.offset
-                continue
-            if last_was_extarg:
-
-                # j = self.stmts.index(inst.offset)
-                # self.lines[j] = offset
-
-                new_inst= inst._replace(starts_line=starts_line,
-                                        is_jump_target=is_jump_target,
-                                        offset=offset)
-                inst = new_inst
-                if i < n:
-                    new_prev = self.prev_op[instructions[i].offset]
-                    j = instructions[i+1].offset
-                    old_prev = self.prev_op[j]
-                    while self.prev_op[j] == old_prev and j < n:
-                        self.prev_op[j] = new_prev
-                        j += 1
-
-            last_was_extarg = False
-            new_instructions.append(inst)
-        return new_instructions
-
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
         """
         Pick out tokens from an uncompyle6 code object, and transform them,
@@ -211,15 +179,15 @@ class Scanner3(Scanner):
         cause specific rules for the specific number of arguments they take.
         """
 
-        # FIXME: remove this when all subsidiary functions have been removed.
-        # We should be able to get everything from the self.insts list.
-        self.code = array('B', co.co_code)
-
-        bytecode = Bytecode(co, self.opc)
         if not show_asm:
             show_asm = self.show_asm
 
-        # show_asm = 'both'
+        if not show_asm:
+            show_asm = self.show_asm
+
+        bytecode = self.build_instructions(co)
+
+        # show_asm = 'after'
         if show_asm in ('both', 'before'):
             for instr in bytecode.get_instructions(co):
                 print(instr.disassemble())
@@ -233,22 +201,14 @@ class Scanner3(Scanner):
         if self.is_pypy:
             customize['PyPy'] = 0
 
-        self.lines = self.build_lines_data(co)
-        self.build_prev_op()
-
-        # FIXME: put as its own method?
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        self.insts = self.remove_extended_args(list(bytecode))
 
-        self.offset2inst_index = {}
         n = len(self.insts)
         for i, inst in enumerate(self.insts):
 
-            self.offset2inst_index[inst.offset] = i
-
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -488,53 +448,6 @@ class Scanner3(Scanner):
             print()
         return tokens, customize
 
-    def build_lines_data(self, code_obj):
-        """
-        Generate various line-related helper data.
-        """
-        # Offset: lineno pairs, only for offsets which start line.
-        # Locally we use list for more convenient iteration using indices
-        linestarts = list(self.opc.findlinestarts(code_obj))
-        self.linestarts = dict(linestarts)
-        # Plain set with offsets of first ops on line
-        self.linestart_offsets = set(a for (a, _) in linestarts)
-        # 'List-map' which shows line number of current op and offset of
-        # first op on following line, given offset of op as index
-        lines = []
-        LineTuple = namedtuple('LineTuple', ['l_no', 'next'])
-        # Iterate through available linestarts, and fill
-        # the data for all code offsets encountered until
-        # last linestart offset
-        _, prev_line_no = linestarts[0]
-        offset = 0
-        for start_offset, line_no in linestarts[1:]:
-            while offset < start_offset:
-                lines.append(LineTuple(prev_line_no, start_offset))
-                offset += 1
-            prev_line_no = line_no
-        # Fill remaining offsets with reference to last line number
-        # and code length as start offset of following non-existing line
-        codelen = len(self.code)
-        while offset < codelen:
-            lines.append(LineTuple(prev_line_no, codelen))
-            offset += 1
-        return lines
-
-    def build_prev_op(self):
-        """
-        Compose 'list-map' which allows to jump to previous
-        op, given offset of current op as index.
-        """
-        code = self.code
-        codelen = len(code)
-        # 2.x uses prev 3.x uses prev_op. Sigh
-        # Until we get this sorted out.
-        self.prev = self.prev_op = [0]
-        for offset in self.op_range(0, codelen):
-            op = code[offset]
-            for _ in range(instruction_size(op, self.opc)):
-                self.prev_op.append(offset)
-
     def find_jump_targets(self, debug):
         """
         Detect all offsets in a byte code which are jump targets

uncompyle6/semantics/customize.py

@@ -17,18 +17,10 @@
 """
 
 from uncompyle6.semantics.consts import (
-    PRECEDENCE, INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT)
+    TABLE_R, TABLE_DIRECT)
 
-from uncompyle6.semantics.make_function import (
-    make_function3_annotate,
-    )
-
-from xdis.util import COMPILER_FLAG_BIT
-from xdis.code import iscode
 from uncompyle6.parsers.astnode import AST
 from uncompyle6.scanners.tok import Token
-from uncompyle6.semantics.helper import flatten_list
-from spark_parser.ast import GenericASTTraversalPruningException
 
 def customize_for_version(self, is_pypy, version):
     if is_pypy:
@@ -189,635 +181,6 @@ def customize_for_version(self, is_pypy, version):
         })
 
     if  version >= 3.0:
-        TABLE_DIRECT.update({
-            'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
-            'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
-            })
-
-        def n_mkfunc_annotate(node):
-
-            if self.version >= 3.3 or node[-2] == 'kwargs':
-                # LOAD_CONST code object ..
-                # LOAD_CONST        'x0'  if >= 3.3
-                # EXTENDED_ARG
-                # MAKE_FUNCTION ..
-                code = node[-4]
-            elif node[-3] == 'expr':
-                code = node[-3][0]
-            else:
-                # LOAD_CONST code object ..
-                # MAKE_FUNCTION ..
-                code = node[-3]
-
-            self.indent_more()
-            for annotate_last in range(len(node)-1, -1, -1):
-                if node[annotate_last] == 'annotate_tuple':
-                    break
-
-            # FIXME: the real situation is that when derived from
-            # function_def_annotate we the name has been filled in.
-            # But when derived from funcdefdeco it hasn't Would like a better
-            # way to distinquish.
-            if self.f.getvalue()[-4:] == 'def ':
-                self.write(code.attr.co_name)
-
-            # FIXME: handle and pass full annotate args
-            make_function3_annotate(self, node, is_lambda=False,
-                                    codeNode=code, annotate_last=annotate_last)
-
-            if len(self.param_stack) > 1:
-                self.write('\n\n')
-            else:
-                self.write('\n\n\n')
-            self.indent_less()
-            self.prune() # stop recursing
-        self.n_mkfunc_annotate = n_mkfunc_annotate
-
-        if version >= 3.4:
-            ########################
-            # Python 3.4+ Additions
-            #######################
-            TABLE_DIRECT.update({
-                'LOAD_CLASSDEREF':	( '%{pattr}', ),
-                })
-
-            if version == 3.4:
-                def n_call(node):
-                    mapping = self._get_mapping(node)
-                    key = node
-                    for i in mapping[1:]:
-                        key = key[i]
-                        pass
-                    if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
-                        # We may want to fill this in...
-                        # But it is distinct from CALL_FUNCTION_VAR below
-                        pass
-                    elif key.kind.startswith('CALL_FUNCTION_VAR'):
-                        # CALL_FUNCTION_VAR's top element of the stack contains
-                        # the variable argument list, then comes
-                        # annotation args, then keyword args.
-                        # In the most least-top-most stack entry, but position 1
-                        # in node order, the positional args.
-                        argc = node[-1].attr
-                        nargs = argc & 0xFF
-                        kwargs = (argc >> 8) & 0xFF
-                        # FIXME: handle annotation args
-                        if kwargs != 0:
-                            # kwargs == 0 is handled by the table entry
-                            # Should probably handle it here though.
-                            if nargs == 0:
-                                template = ('%c(*%c, %C)',
-                                            0, -2, (1, kwargs+1, ', '))
-                            else:
-                                template = ('%c(%C, *%c, %C)',
-                                            0, (1, nargs+1, ', '),
-                                            -2, (-2-kwargs, -2, ', '))
-                            self.template_engine(template, node)
-                            self.prune()
-
-                    self.default(node)
-                self.n_call = n_call
-
-
-            ########################
-            # Python 3.5+ Additions
-            #######################
-            if version >= 3.5:
-                TABLE_DIRECT.update({
-                    'await_expr':	       ( 'await %c', 0),
-                    'await_stmt':	       ( '%|%c\n', 0),
-                    'async_for_stmt':      (
-                        '%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
-                    'async_forelse_stmt':  (
-                        '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
-                    'async_with_stmt':     (
-                        '%|async with %c:\n%+%c%-', 0, 7),
-                    'async_with_as_stmt':  (
-                        '%|async with %c as %c:\n%+%c%-', 0, 6, 7),
-                    'unmap_dict':	       ( '{**%C}', (0, -1, ', **') ),
-                    # 'unmapexpr':	       ( '{**%c}', 0), # done by n_unmapexpr
-
-                })
-
-                def async_call(node):
-                    self.f.write('async ')
-                    node.kind == 'call'
-                    p = self.prec
-                    self.prec = 80
-                    self.template_engine(('%c(%P)', 0, (1, -4, ', ',
-                                          100)), node)
-                    self.prec = p
-                    node.kind == 'async_call'
-                    self.prune()
-                self.n_async_call = async_call
-                self.n_build_list_unpack = self.n_list
-
-                if version == 3.5:
-                    def n_call(node):
-                        mapping = self._get_mapping(node)
-                        table = mapping[0]
-                        key = node
-                        for i in mapping[1:]:
-                            key = key[i]
-                            pass
-                        if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
-                            # Python 3.5 changes the stack position of
-                            # *args: kwargs come after *args whereas
-                            # in earlier Pythons, *args is at the end
-                            # which simplifies things from our
-                            # perspective.  Python 3.6+ replaces
-                            # CALL_FUNCTION_VAR_KW with
-                            # CALL_FUNCTION_EX We will just swap the
-                            # order to make it look like earlier
-                            # Python 3.
-                            entry = table[key.kind]
-                            kwarg_pos = entry[2][1]
-                            args_pos = kwarg_pos - 1
-                            # Put last node[args_pos] after subsequent kwargs
-                            while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
-                                # swap node[args_pos] with node[kwargs_pos]
-                                node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
-                                args_pos = kwarg_pos
-                                kwarg_pos += 1
-                        elif key.kind.startswith('CALL_FUNCTION_VAR'):
-                            # CALL_FUNCTION_VAR's top element of the stack contains
-                            # the variable argument list, then comes
-                            # annotation args, then keyword args.
-                            # In the most least-top-most stack entry, but position 1
-                            # in node order, the positional args.
-                            argc = node[-1].attr
-                            nargs = argc & 0xFF
-                            kwargs = (argc >> 8) & 0xFF
-                            # FIXME: handle annotation args
-                            if nargs > 0:
-                                template = ('%c(%C, ', 0, (1, nargs+1, ', '))
-                            else:
-                                template = ('%c(', 0)
-                            self.template_engine(template, node)
-
-                            args_node =  node[-2]
-                            if args_node in ('pos_arg', 'expr'):
-                                args_node = args_node[0]
-                            if args_node == 'build_list_unpack':
-                                template = ('*%P)', (0, len(args_node)-1, ', *', 100))
-                                self.template_engine(template, args_node)
-                            else:
-                                if len(node) - nargs > 3:
-                                    template = ('*%c, %C)', 1, (nargs+kwargs+1, -1, ', '))
-                                else:
-                                    template = ('*%c)', 1)
-                                self.template_engine(template, node)
-                            self.prune()
-
-                        self.default(node)
-                    self.n_call = n_call
-
-                def n_function_def(node):
-                    if self.version == 3.6:
-                        code_node = node[0][0]
-                    else:
-                        code_node = node[0][1]
-
-                    is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
-                    if (is_code and
-                        (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
-                        self.template_engine(('\n\n%|async def %c\n',
-                                              -2), node)
-                    else:
-                        self.template_engine(('\n\n%|def %c\n', -2),
-                                             node)
-                    self.prune()
-                self.n_function_def = n_function_def
-
-                def unmapexpr(node):
-                    last_n = node[0][-1]
-                    for n in node[0]:
-                        self.preorder(n)
-                        if n != last_n:
-                            self.f.write(', **')
-                            pass
-                        pass
-                    self.prune()
-                    pass
-                self.n_unmapexpr = unmapexpr
-
-            if version >= 3.6:
-                ########################
-                # Python 3.6+ Additions
-                #######################
-
-                # Value 100 is important; it is exactly
-                # module/function precidence.
-                PRECEDENCE['call_kw']     = 100
-                PRECEDENCE['call_kw36']   = 100
-                PRECEDENCE['call_ex']     = 100
-                PRECEDENCE['call_ex_kw']  = 100
-                PRECEDENCE['call_ex_kw2'] = 100
-                PRECEDENCE['call_ex_kw3'] = 100
-                PRECEDENCE['call_ex_kw4'] = 100
-
-                TABLE_DIRECT.update({
-                    'tryfinally36':  ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
-                                       (1, 'returns'), 3 ),
-                    'fstring_expr':   ( "{%c%{conversion}}", 0),
-                    # FIXME: the below assumes the format strings
-                    # don't have ''' in them. Fix this properly
-                    'fstring_single': ( "f'''{%c%{conversion}}'''", 0),
-                    'fstring_multi':  ( "f'''%c'''", 0),
-                    'func_args36':    ( "%c(**", 0),
-                    'try_except36':   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
-                    'except_return':  ( '%|except:\n%+%c%-', 3 ),
-                    'unpack_list':    ( '*%c', (0, 'list') ),
-                    'call_ex' : (
-                        '%c(%p)',
-                        (0, 'expr'), (1, 100)),
-                    'call_ex_kw' : (
-                        '%c(%p)',
-                        (0, 'expr'), (2, 100)),
-
-                })
-
-                TABLE_R.update({
-                    'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)),
-                    # Not quite right
-                    'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')),
-                    })
-
-                def build_unpack_tuple_with_call(node):
-
-                    if node[0] == 'expr':
-                        tup = node[0][0]
-                    else:
-                        tup = node[0]
-                        pass
-                    assert tup == 'tuple'
-                    self.call36_tuple(tup)
-
-                    buwc = node[-1]
-                    assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
-                    for n in node[1:-1]:
-                        self.f.write(', *')
-                        self.preorder(n)
-                        pass
-                    self.prune()
-                    return
-                self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call
-
-                def build_unpack_map_with_call(node):
-                    n = node[0]
-                    if n == 'expr':
-                        n = n[0]
-                    if n == 'dict':
-                        self.call36_dict(n)
-                        first = 1
-                        sep = ', **'
-                    else:
-                        first = 0
-                        sep = '**'
-                    for n in node[first:-1]:
-                        self.f.write(sep)
-                        self.preorder(n)
-                        sep = ', **'
-                        pass
-                    self.prune()
-                    return
-                self.n_build_map_unpack_with_call = build_unpack_map_with_call
-
-                def call_ex_kw2(node):
-                    """Handle CALL_FUNCTION_EX 2  (have KW) but with
-                    BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
-
-                    # This is weird shit. Thanks Python!
-                    self.preorder(node[0])
-                    self.write('(')
-
-                    assert node[1] == 'build_tuple_unpack_with_call'
-                    btuwc = node[1]
-                    tup = btuwc[0]
-                    if tup == 'expr':
-                        tup = tup[0]
-                    assert tup == 'tuple'
-                    self.call36_tuple(tup)
-                    assert node[2] == 'build_map_unpack_with_call'
-
-                    self.write(', ')
-                    d = node[2][0]
-                    if d == 'expr':
-                        d = d[0]
-                    assert d == 'dict'
-                    self.call36_dict(d)
-
-                    args = btuwc[1]
-                    self.write(', *')
-                    self.preorder(args)
-
-                    self.write(', **')
-                    star_star_args = node[2][1]
-                    if star_star_args == 'expr':
-                        star_star_args = star_star_args[0]
-                    self.preorder(star_star_args)
-                    self.write(')')
-                    self.prune()
-                self.n_call_ex_kw2 = call_ex_kw2
-
-                def call_ex_kw3(node):
-                    """Handle CALL_FUNCTION_EX 1 (have KW) but without
-                    BUILD_MAP_UNPACK_WITH_CALL"""
-                    self.preorder(node[0])
-                    self.write('(')
-                    args = node[1][0]
-                    if args == 'expr':
-                        args = args[0]
-                    if args == 'tuple':
-                        if self.call36_tuple(args) > 0:
-                            self.write(', ')
-                            pass
-                        pass
-
-                    self.write('*')
-                    self.preorder(node[1][1])
-                    self.write(', ')
-
-                    kwargs = node[2]
-                    if kwargs == 'expr':
-                        kwargs = kwargs[0]
-                    if kwargs == 'dict':
-                        self.call36_dict(kwargs)
-                    else:
-                        self.write('**')
-                        self.preorder(kwargs)
-                    self.write(')')
-                    self.prune()
-                self.n_call_ex_kw3 = call_ex_kw3
-
-                def call_ex_kw4(node):
-                    """Handle CALL_FUNCTION_EX {1 or 2} but without
-                    BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
-                    self.preorder(node[0])
-                    self.write('(')
-                    args = node[1][0]
-                    if args == 'tuple':
-                        if self.call36_tuple(args) > 0:
-                            self.write(', ')
-                            pass
-                        pass
-                    else:
-                        self.write('*')
-                        self.preorder(args)
-                        self.write(', ')
-                        pass
-
-                    kwargs = node[2]
-                    if kwargs == 'expr':
-                        kwargs = kwargs[0]
-                    call_function_ex = node[-1]
-                    assert call_function_ex == 'CALL_FUNCTION_EX_KW'
-                    # FIXME: decide if the below test be on kwargs == 'dict'
-                    if (call_function_ex.attr & 1 and
-                        (not isinstance(kwargs, Token) and kwargs != 'attribute')
-                        and not kwargs[0].kind.startswith('kvlist')):
-                        self.call36_dict(kwargs)
-                    else:
-                        self.write('**')
-                        self.preorder(kwargs)
-                    self.write(')')
-                    self.prune()
-                self.n_call_ex_kw4 = call_ex_kw4
-
-                def call36_tuple(node):
-                    """
-                    A tuple used in a call, these are like normal tuples but they
-                    don't have the enclosing parenthesis.
-                    """
-                    assert node == 'tuple'
-                    # Note: don't iterate over last element which is a
-                    # BUILD_TUPLE...
-                    flat_elems = flatten_list(node[:-1])
-
-                    self.indent_more(INDENT_PER_LEVEL)
-                    sep = ''
-
-                    for elem in flat_elems:
-                        if elem in ('ROT_THREE', 'EXTENDED_ARG'):
-                            continue
-                        assert elem == 'expr'
-                        line_number = self.line_number
-                        value = self.traverse(elem)
-                        if line_number != self.line_number:
-                            sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
-                        self.write(sep, value)
-                        sep = ', '
-
-                    self.indent_less(INDENT_PER_LEVEL)
-                    return len(flat_elems)
-                self.call36_tuple = call36_tuple
-
-                def call36_dict(node):
-                    """
-                    A dict used in a call_ex_kw2, which are a dictionary items expressed
-                    in a call. This should format to:
-                         a=1, b=2
-                    In other words, no braces, no quotes around keys and ":" becomes
-                    "=".
-
-                    We will source-code use line breaks to guide us when to break.
-                    """
-                    p = self.prec
-                    self.prec = 100
-
-                    self.indent_more(INDENT_PER_LEVEL)
-                    sep = INDENT_PER_LEVEL[:-1]
-                    line_number = self.line_number
-
-                    if  node[0].kind.startswith('kvlist'):
-                        # Python 3.5+ style key/value list in dict
-                        kv_node = node[0]
-                        l = list(kv_node)
-                        i = 0
-
-                        length = len(l)
-                        # FIXME: Parser-speed improved grammars will have BUILD_MAP
-                        # at the end. So in the future when everything is
-                        # complete, we can do an "assert" instead of "if".
-                        if kv_node[-1].kind.startswith("BUILD_MAP"):
-                            length -= 1
-
-                        # Respect line breaks from source
-                        while i < length:
-                            self.write(sep)
-                            name = self.traverse(l[i], indent='')
-                            # Strip off beginning and trailing quotes in name
-                            name = name[1:-1]
-                            if i > 0:
-                                line_number = self.indent_if_source_nl(line_number,
-                                                                       self.indent + INDENT_PER_LEVEL[:-1])
-                            line_number = self.line_number
-                            self.write(name, '=')
-                            value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
-                            self.write(value)
-                            sep = ", "
-                            if line_number != self.line_number:
-                                sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
-                                line_number = self.line_number
-                            i += 2
-                            pass
-                    elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
-                        keys_node = node[-2]
-                        keys = keys_node.attr
-                        # from trepan.api import debug; debug()
-                        assert keys_node == 'LOAD_CONST' and isinstance(keys, tuple)
-                        for i in range(node[-1].attr):
-                            self.write(sep)
-                            self.write(keys[i], '=')
-                            value = self.traverse(node[i], indent='')
-                            self.write(value)
-                            sep = ", "
-                            if line_number != self.line_number:
-                                sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
-                                line_number = self.line_number
-                                pass
-                            pass
-                    else:
-                        self.write("**")
-                        try:
-                            self.default(node)
-                        except GenericASTTraversalPruningException:
-                            pass
-
-                    self.prec = p
-                    self.indent_less(INDENT_PER_LEVEL)
-                    return
-                self.call36_dict = call36_dict
-
-
-                FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
-
-                def n_except_suite_finalize(node):
-                    if node[1] == 'returns' and self.hide_internal:
-                        # Process node[1] only.
-                        # The code after "returns", e.g. node[3], is dead code.
-                        # Adding it is wrong as it dedents and another
-                        # exception handler "except_stmt" afterwards.
-                        # Note it is also possible that the grammar is wrong here.
-                        # and this should not be "except_stmt".
-                        self.indent_more()
-                        self.preorder(node[1])
-                        self.indent_less()
-                    else:
-                        self.default(node)
-                    self.prune()
-                self.n_except_suite_finalize = n_except_suite_finalize
-
-                def n_formatted_value(node):
-                    if node[0] == 'LOAD_CONST':
-                        self.write(node[0].attr)
-                        self.prune()
-                    else:
-                        self.default(node)
-                self.n_formatted_value = n_formatted_value
-
-                def f_conversion(node):
-                    node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '')
-
-                def fstring_expr(node):
-                    f_conversion(node)
-                    self.default(node)
-                self.n_fstring_expr = fstring_expr
-
-                def fstring_single(node):
-                    f_conversion(node)
-                    self.default(node)
-                self.n_fstring_single = fstring_single
-
-                # def kwargs_only_36(node):
-                #     keys = node[-1].attr
-                #     num_kwargs = len(keys)
-                #     values = node[:num_kwargs]
-                #     for i, (key, value) in enumerate(zip(keys, values)):
-                #         self.write(key + '=')
-                #         self.preorder(value)
-                #         if i < num_kwargs:
-                #             self.write(',')
-                #     self.prune()
-                #     return
-                # self.n_kwargs_only_36 = kwargs_only_36
-
-                def n_call_kw36(node):
-                    self.template_engine(("%c(", 0), node)
-                    keys = node[-2].attr
-                    num_kwargs = len(keys)
-                    num_posargs = len(node) - (num_kwargs + 2)
-                    n = len(node)
-                    assert n >= len(keys)+1, \
-                      'not enough parameters keyword-tuple values'
-                    sep = ''
-
-                    line_number = self.line_number
-                    for i in range(1, num_posargs):
-                        self.write(sep)
-                        self.preorder(node[i])
-                        if line_number != self.line_number:
-                            sep = ",\n" + self.indent + "  "
-                        else:
-                            sep = ", "
-                        line_number = self.line_number
-
-                    i = num_posargs
-                    j = 0
-                    # FIXME: adjust output for line breaks?
-                    while i < n-2:
-                        self.write(sep)
-                        self.write(keys[j] + '=')
-                        self.preorder(node[i])
-                        if line_number != self.line_number:
-                            sep = ",\n" + self.indent + "  "
-                        else:
-                            sep = ", "
-                        i += 1
-                        j += 1
-                    self.write(')')
-                    self.prune()
-                    return
-                self.n_call_kw36 = n_call_kw36
-
-                def starred(node):
-                    l = len(node)
-                    assert l > 0
-                    pos_args = node[0]
-                    if pos_args == 'expr':
-                        pos_args = pos_args[0]
-                    if pos_args == 'tuple':
-                        build_tuple = pos_args[0]
-                        if build_tuple.kind.startswith('BUILD_TUPLE'):
-                            tuple_len = 0
-                        else:
-                            tuple_len = len(node) - 1
-                        star_start = 1
-                        template = '%C', (0, -1, ', ')
-                        self.template_engine(template, pos_args)
-                        if tuple_len == 0:
-                            self.write("*()")
-                            # That's it
-                            self.prune()
-                        self.write(', ')
-                    else:
-                        star_start = 0
-                    if l > 1:
-                        template = ( '*%C', (star_start, -1, ', *') )
-                    else:
-                        template = ( '*%c', (star_start, 'expr') )
-
-                    self.template_engine(template, node)
-                    self.prune()
-
-                self.n_starred = starred
-
-                def return_closure(node):
-                    # Nothing should be output here
-                    self.prune()
-                    return
-                self.n_return_closure = return_closure
-                pass # version >= 3.6
-            pass # version >= 3.4
-        pass # version >= 3.0
+        from uncompyle6.semantics.customize3 import customize_for_version3
+        customize_for_version3(self, version)
     return

uncompyle6/semantics/customize3.py

@@ -0,0 +1,759 @@
+#  Copyright (c) 2018 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Isolate Python 3 version-specific semantic actions here.
+"""
+
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT, TABLE_R)
+
+from xdis.code import iscode
+from xdis.util import COMPILER_FLAG_BIT
+from spark_parser.ast import GenericASTTraversalPruningException
+from uncompyle6.scanners.tok import Token
+from uncompyle6.semantics.helper import flatten_list
+from uncompyle6.semantics.make_function import make_function3_annotate
+
+def customize_for_version3(self, version):
+    TABLE_DIRECT.update({
+        'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
+        'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
+        })
+
+    if version >= 3.3:
+        def n_yield_from(node):
+            self.write('yield from')
+            self.write(' ')
+            if 3.3 <= self.version <= 3.4:
+                self.preorder(node[0][0][0][0])
+            elif self.version >= 3.5:
+                self.preorder(node[0])
+            else:
+                assert False, "dunno about this python version"
+            self.prune() # stop recursing
+        self.n_yield_from = n_yield_from
+
+    if 3.2 <= version <= 3.4:
+        def n_call(node):
+            mapping = self._get_mapping(node)
+            key = node
+            for i in mapping[1:]:
+                key = key[i]
+                pass
+            if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
+                # We may want to fill this in...
+                # But it is distinct from CALL_FUNCTION_VAR below
+                pass
+            elif key.kind.startswith('CALL_FUNCTION_VAR'):
+                # CALL_FUNCTION_VAR's top element of the stack contains
+                # the variable argument list, then comes
+                # annotation args, then keyword args.
+                # In the most least-top-most stack entry, but position 1
+                # in node order, the positional args.
+                argc = node[-1].attr
+                nargs = argc & 0xFF
+                kwargs = (argc >> 8) & 0xFF
+                # FIXME: handle annotation args
+                if kwargs != 0:
+                    # kwargs == 0 is handled by the table entry
+                    # Should probably handle it here though.
+                    if nargs == 0:
+                        template = ('%c(*%c, %C)',
+                                    0, -2, (1, kwargs+1, ', '))
+                    else:
+                        template = ('%c(%C, *%c, %C)',
+                                    0, (1, nargs+1, ', '),
+                                    -2, (-2-kwargs, -2, ', '))
+                    self.template_engine(template, node)
+                    self.prune()
+
+            self.default(node)
+        self.n_call = n_call
+
+
+    def n_mkfunc_annotate(node):
+
+        if self.version >= 3.3 or node[-2] == 'kwargs':
+            # LOAD_CONST code object ..
+            # LOAD_CONST        'x0'  if >= 3.3
+            # EXTENDED_ARG
+            # MAKE_FUNCTION ..
+            code = node[-4]
+        elif node[-3] == 'expr':
+            code = node[-3][0]
+        else:
+            # LOAD_CONST code object ..
+            # MAKE_FUNCTION ..
+            code = node[-3]
+
+        self.indent_more()
+        for annotate_last in range(len(node)-1, -1, -1):
+            if node[annotate_last] == 'annotate_tuple':
+                break
+
+        # FIXME: the real situation is that when derived from
+        # function_def_annotate we the name has been filled in.
+        # But when derived from funcdefdeco it hasn't Would like a better
+        # way to distinquish.
+        if self.f.getvalue()[-4:] == 'def ':
+            self.write(code.attr.co_name)
+
+        # FIXME: handle and pass full annotate args
+        make_function3_annotate(self, node, is_lambda=False,
+                                codeNode=code, annotate_last=annotate_last)
+
+        if len(self.param_stack) > 1:
+            self.write('\n\n')
+        else:
+            self.write('\n\n\n')
+        self.indent_less()
+        self.prune() # stop recursing
+    self.n_mkfunc_annotate = n_mkfunc_annotate
+
+    if version >= 3.4:
+        ########################
+        # Python 3.4+ Additions
+        #######################
+        TABLE_DIRECT.update({
+            'LOAD_CLASSDEREF':	( '%{pattr}', ),
+            })
+
+        ########################
+        # Python 3.5+ Additions
+        #######################
+        if version >= 3.5:
+            TABLE_DIRECT.update({
+                'await_expr':	       ( 'await %c', 0),
+                'await_stmt':	       ( '%|%c\n', 0),
+                'async_for_stmt':      (
+                    '%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
+                'async_forelse_stmt':  (
+                    '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
+                'async_with_stmt':     (
+                    '%|async with %c:\n%+%c%-', 0, 7),
+                'async_with_as_stmt':  (
+                    '%|async with %c as %c:\n%+%c%-', 0, 6, 7),
+                'unmap_dict':	       ( '{**%C}', (0, -1, ', **') ),
+                # 'unmapexpr':	       ( '{**%c}', 0), # done by n_unmapexpr
+
+            })
+
+            def async_call(node):
+                self.f.write('async ')
+                node.kind == 'call'
+                p = self.prec
+                self.prec = 80
+                self.template_engine(('%c(%P)', 0, (1, -4, ', ',
+                                      100)), node)
+                self.prec = p
+                node.kind == 'async_call'
+                self.prune()
+            self.n_async_call = async_call
+            self.n_build_list_unpack = self.n_list
+
+            if version == 3.5:
+                def n_call(node):
+                    mapping = self._get_mapping(node)
+                    table = mapping[0]
+                    key = node
+                    for i in mapping[1:]:
+                        key = key[i]
+                        pass
+                    if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
+                        # Python 3.5 changes the stack position of
+                        # *args: kwargs come after *args whereas
+                        # in earlier Pythons, *args is at the end
+                        # which simplifies things from our
+                        # perspective.  Python 3.6+ replaces
+                        # CALL_FUNCTION_VAR_KW with
+                        # CALL_FUNCTION_EX We will just swap the
+                        # order to make it look like earlier
+                        # Python 3.
+                        entry = table[key.kind]
+                        kwarg_pos = entry[2][1]
+                        args_pos = kwarg_pos - 1
+                        # Put last node[args_pos] after subsequent kwargs
+                        while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
+                            # swap node[args_pos] with node[kwargs_pos]
+                            node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
+                            args_pos = kwarg_pos
+                            kwarg_pos += 1
+                    elif key.kind.startswith('CALL_FUNCTION_VAR'):
+                        # CALL_FUNCTION_VAR's top element of the stack contains
+                        # the variable argument list, then comes
+                        # annotation args, then keyword args.
+                        # In the most least-top-most stack entry, but position 1
+                        # in node order, the positional args.
+                        argc = node[-1].attr
+                        nargs = argc & 0xFF
+                        kwargs = (argc >> 8) & 0xFF
+                        # FIXME: handle annotation args
+                        if nargs > 0:
+                            template = ('%c(%C, ', 0, (1, nargs+1, ', '))
+                        else:
+                            template = ('%c(', 0)
+                        self.template_engine(template, node)
+
+                        args_node =  node[-2]
+                        if args_node in ('pos_arg', 'expr'):
+                            args_node = args_node[0]
+                        if args_node == 'build_list_unpack':
+                            template = ('*%P)', (0, len(args_node)-1, ', *', 100))
+                            self.template_engine(template, args_node)
+                        else:
+                            if len(node) - nargs > 3:
+                                template = ('*%c, %C)', nargs+1, (nargs+kwargs+1, -1, ', '))
+                            else:
+                                template = ('*%c)', nargs+1)
+                            self.template_engine(template, node)
+                        self.prune()
+
+                    self.default(node)
+                self.n_call = n_call
+
+            def n_function_def(node):
+                if self.version == 3.6:
+                    code_node = node[0][0]
+                else:
+                    code_node = node[0][1]
+
+                is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
+                if (is_code and
+                    (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
+                    self.template_engine(('\n\n%|async def %c\n',
+                                          -2), node)
+                else:
+                    self.template_engine(('\n\n%|def %c\n', -2),
+                                         node)
+                self.prune()
+            self.n_function_def = n_function_def
+
+            def unmapexpr(node):
+                last_n = node[0][-1]
+                for n in node[0]:
+                    self.preorder(n)
+                    if n != last_n:
+                        self.f.write(', **')
+                        pass
+                    pass
+                self.prune()
+                pass
+            self.n_unmapexpr = unmapexpr
+
+            # FIXME: start here
+            def n_list_unpack(node):
+                """
+                prettyprint an unpacked list or tuple
+                """
+                p = self.prec
+                self.prec = 100
+                lastnode = node.pop()
+                lastnodetype = lastnode.kind
+
+                # If this build list is inside a CALL_FUNCTION_VAR,
+                # then the first * has already been printed.
+                # Until I have a better way to check for CALL_FUNCTION_VAR,
+                # will assume that if the text ends in *.
+                last_was_star = self.f.getvalue().endswith('*')
+
+                if lastnodetype.startswith('BUILD_LIST'):
+                    self.write('['); endchar = ']'
+                elif lastnodetype.startswith('BUILD_TUPLE'):
+                    # Tuples can appear places that can NOT
+                    # have parenthesis around them, like array
+                    # subscripts. We check for that by seeing
+                    # if a tuple item is some sort of slice.
+                    no_parens = False
+                    for n in node:
+                        if n == 'expr' and n[0].kind.startswith('build_slice'):
+                            no_parens = True
+                            break
+                        pass
+                    if no_parens:
+                        endchar = ''
+                    else:
+                        self.write('('); endchar = ')'
+                        pass
+
+                elif lastnodetype.startswith('BUILD_SET'):
+                    self.write('{'); endchar = '}'
+                elif lastnodetype.startswith('BUILD_MAP_UNPACK'):
+                    self.write('{*'); endchar = '}'
+                elif lastnodetype.startswith('ROT_TWO'):
+                    self.write('('); endchar = ')'
+                else:
+                    raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')
+
+                flat_elems = flatten_list(node)
+
+                self.indent_more(INDENT_PER_LEVEL)
+                sep = ''
+                for elem in flat_elems:
+                    if elem in ('ROT_THREE', 'EXTENDED_ARG'):
+                        continue
+                    assert elem == 'expr'
+                    line_number = self.line_number
+                    value = self.traverse(elem)
+                    if elem[0] == 'tuple':
+                        assert value[0] == '('
+                        assert value[-1] == ')'
+                        value = value[1:-1]
+                        if value[-1] == ',':
+                            # singleton tuple
+                            value = value[:-1]
+                    else:
+                        value = '*' + value
+                    if line_number != self.line_number:
+                        sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
+                    else:
+                        if sep != '': sep += ' '
+                    if not last_was_star:
+                        pass
+                    else:
+                        last_was_star = False
+                    self.write(sep, value)
+                    sep = ','
+                if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
+                    self.write(',')
+                self.write(endchar)
+                self.indent_less(INDENT_PER_LEVEL)
+
+                self.prec = p
+                self.prune()
+                return
+
+            self.n_tuple_unpack = n_list_unpack
+
+
+        if version >= 3.6:
+            ########################
+            # Python 3.6+ Additions
+            #######################
+
+            # Value 100 is important; it is exactly
+            # module/function precidence.
+            PRECEDENCE['call_kw']     = 100
+            PRECEDENCE['call_kw36']   = 100
+            PRECEDENCE['call_ex']     = 100
+            PRECEDENCE['call_ex_kw']  = 100
+            PRECEDENCE['call_ex_kw2'] = 100
+            PRECEDENCE['call_ex_kw3'] = 100
+            PRECEDENCE['call_ex_kw4'] = 100
+            PRECEDENCE['unmap_dict']  = 0
+
+            TABLE_DIRECT.update({
+                'tryfinally36':  ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
+                                   (1, 'returns'), 3 ),
+                'fstring_expr':   ( "{%c%{conversion}}", 0),
+                # FIXME: the below assumes the format strings
+                # don't have ''' in them. Fix this properly
+                'fstring_single': ( "f'''{%c%{conversion}}'''", 0),
+                'fstring_multi':  ( "f'''%c'''", 0),
+                'func_args36':    ( "%c(**", 0),
+                'try_except36':   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
+                'except_return':  ( '%|except:\n%+%c%-', 3 ),
+                'unpack_list':    ( '*%c', (0, 'list') ),
+                'call_ex' : (
+                    '%c(%p)',
+                    (0, 'expr'), (1, 100)),
+                'call_ex_kw' : (
+                    '%c(%p)',
+                    (0, 'expr'), (2, 100)),
+
+            })
+
+            TABLE_R.update({
+                'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)),
+                # Not quite right
+                'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')),
+                })
+
+            def build_unpack_tuple_with_call(node):
+
+                if node[0] == 'expr':
+                    tup = node[0][0]
+                else:
+                    tup = node[0]
+                    pass
+                assert tup == 'tuple'
+                self.call36_tuple(tup)
+
+                buwc = node[-1]
+                assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
+                for n in node[1:-1]:
+                    self.f.write(', *')
+                    self.preorder(n)
+                    pass
+                self.prune()
+                return
+            self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call
+
+            def build_unpack_map_with_call(node):
+                n = node[0]
+                if n == 'expr':
+                    n = n[0]
+                if n == 'dict':
+                    self.call36_dict(n)
+                    first = 1
+                    sep = ', **'
+                else:
+                    first = 0
+                    sep = '**'
+                for n in node[first:-1]:
+                    self.f.write(sep)
+                    self.preorder(n)
+                    sep = ', **'
+                    pass
+                self.prune()
+                return
+            self.n_build_map_unpack_with_call = build_unpack_map_with_call
+
+            def call_ex_kw2(node):
+                """Handle CALL_FUNCTION_EX 2  (have KW) but with
+                BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
+
+                # This is weird shit. Thanks Python!
+                self.preorder(node[0])
+                self.write('(')
+
+                assert node[1] == 'build_tuple_unpack_with_call'
+                btuwc = node[1]
+                tup = btuwc[0]
+                if tup == 'expr':
+                    tup = tup[0]
+                assert tup == 'tuple'
+                self.call36_tuple(tup)
+                assert node[2] == 'build_map_unpack_with_call'
+
+                self.write(', ')
+                d = node[2][0]
+                if d == 'expr':
+                    d = d[0]
+                assert d == 'dict'
+                self.call36_dict(d)
+
+                args = btuwc[1]
+                self.write(', *')
+                self.preorder(args)
+
+                self.write(', **')
+                star_star_args = node[2][1]
+                if star_star_args == 'expr':
+                    star_star_args = star_star_args[0]
+                self.preorder(star_star_args)
+                self.write(')')
+                self.prune()
+            self.n_call_ex_kw2 = call_ex_kw2
+
+            def call_ex_kw3(node):
+                """Handle CALL_FUNCTION_EX 1 (have KW) but without
+                BUILD_MAP_UNPACK_WITH_CALL"""
+                self.preorder(node[0])
+                self.write('(')
+                args = node[1][0]
+                if args == 'expr':
+                    args = args[0]
+                if args == 'tuple':
+                    if self.call36_tuple(args) > 0:
+                        self.write(', ')
+                        pass
+                    pass
+
+                self.write('*')
+                self.preorder(node[1][1])
+                self.write(', ')
+
+                kwargs = node[2]
+                if kwargs == 'expr':
+                    kwargs = kwargs[0]
+                if kwargs == 'dict':
+                    self.call36_dict(kwargs)
+                else:
+                    self.write('**')
+                    self.preorder(kwargs)
+                self.write(')')
+                self.prune()
+            self.n_call_ex_kw3 = call_ex_kw3
+
+            def call_ex_kw4(node):
+                """Handle CALL_FUNCTION_EX {1 or 2} but without
+                BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
+                self.preorder(node[0])
+                self.write('(')
+                args = node[1][0]
+                if args == 'tuple':
+                    if self.call36_tuple(args) > 0:
+                        self.write(', ')
+                        pass
+                    pass
+                else:
+                    self.write('*')
+                    self.preorder(args)
+                    self.write(', ')
+                    pass
+
+                kwargs = node[2]
+                if kwargs == 'expr':
+                    kwargs = kwargs[0]
+                call_function_ex = node[-1]
+                assert call_function_ex == 'CALL_FUNCTION_EX_KW'
+                # FIXME: decide if the below test be on kwargs == 'dict'
+                if (call_function_ex.attr & 1 and
+                    (not isinstance(kwargs, Token) and kwargs != 'attribute')
+                    and not kwargs[0].kind.startswith('kvlist')):
+                    self.call36_dict(kwargs)
+                else:
+                    self.write('**')
+                    self.preorder(kwargs)
+                self.write(')')
+                self.prune()
+            self.n_call_ex_kw4 = call_ex_kw4
+
+            def call36_tuple(node):
+                """
+                A tuple used in a call, these are like normal tuples but they
+                don't have the enclosing parenthesis.
+                """
+                assert node == 'tuple'
+                # Note: don't iterate over last element which is a
+                # BUILD_TUPLE...
+                flat_elems = flatten_list(node[:-1])
+
+                self.indent_more(INDENT_PER_LEVEL)
+                sep = ''
+
+                for elem in flat_elems:
+                    if elem in ('ROT_THREE', 'EXTENDED_ARG'):
+                        continue
+                    assert elem == 'expr'
+                    line_number = self.line_number
+                    value = self.traverse(elem)
+                    if line_number != self.line_number:
+                        sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
+                    self.write(sep, value)
+                    sep = ', '
+
+                self.indent_less(INDENT_PER_LEVEL)
+                return len(flat_elems)
+            self.call36_tuple = call36_tuple
+
+            def call36_dict(node):
+                """
+                A dict used in a call_ex_kw2, which are a dictionary items expressed
+                in a call. This should format to:
+                     a=1, b=2
+                In other words, no braces, no quotes around keys and ":" becomes
+                "=".
+
+                We will source-code use line breaks to guide us when to break.
+                """
+                p = self.prec
+                self.prec = 100
+
+                self.indent_more(INDENT_PER_LEVEL)
+                sep = INDENT_PER_LEVEL[:-1]
+                line_number = self.line_number
+
+                if  node[0].kind.startswith('kvlist'):
+                    # Python 3.5+ style key/value list in dict
+                    kv_node = node[0]
+                    l = list(kv_node)
+                    i = 0
+
+                    length = len(l)
+                    # FIXME: Parser-speed improved grammars will have BUILD_MAP
+                    # at the end. So in the future when everything is
+                    # complete, we can do an "assert" instead of "if".
+                    if kv_node[-1].kind.startswith("BUILD_MAP"):
+                        length -= 1
+
+                    # Respect line breaks from source
+                    while i < length:
+                        self.write(sep)
+                        name = self.traverse(l[i], indent='')
+                        # Strip off beginning and trailing quotes in name
+                        name = name[1:-1]
+                        if i > 0:
+                            line_number = self.indent_if_source_nl(line_number,
+                                                                   self.indent + INDENT_PER_LEVEL[:-1])
+                        line_number = self.line_number
+                        self.write(name, '=')
+                        value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
+                        self.write(value)
+                        sep = ", "
+                        if line_number != self.line_number:
+                            sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
+                            line_number = self.line_number
+                        i += 2
+                        pass
+                elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
+                    keys_node = node[-2]
+                    keys = keys_node.attr
+                    # from trepan.api import debug; debug()
+                    assert keys_node == 'LOAD_CONST' and isinstance(keys, tuple)
+                    for i in range(node[-1].attr):
+                        self.write(sep)
+                        self.write(keys[i], '=')
+                        value = self.traverse(node[i], indent='')
+                        self.write(value)
+                        sep = ", "
+                        if line_number != self.line_number:
+                            sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
+                            line_number = self.line_number
+                            pass
+                        pass
+                else:
+                    self.write("**")
+                    try:
+                        self.default(node)
+                    except GenericASTTraversalPruningException:
+                        pass
+
+                self.prec = p
+                self.indent_less(INDENT_PER_LEVEL)
+                return
+            self.call36_dict = call36_dict
+
+
+            FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
+
+            def n_except_suite_finalize(node):
+                if node[1] == 'returns' and self.hide_internal:
+                    # Process node[1] only.
+                    # The code after "returns", e.g. node[3], is dead code.
+                    # Adding it is wrong as it dedents and another
+                    # exception handler "except_stmt" afterwards.
+                    # Note it is also possible that the grammar is wrong here.
+                    # and this should not be "except_stmt".
+                    self.indent_more()
+                    self.preorder(node[1])
+                    self.indent_less()
+                else:
+                    self.default(node)
+                self.prune()
+            self.n_except_suite_finalize = n_except_suite_finalize
+
+            def n_formatted_value(node):
+                if node[0] == 'LOAD_CONST':
+                    self.write(node[0].attr)
+                    self.prune()
+                else:
+                    self.default(node)
+            self.n_formatted_value = n_formatted_value
+
+            def f_conversion(node):
+                node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '')
+
+            def fstring_expr(node):
+                f_conversion(node)
+                self.default(node)
+            self.n_fstring_expr = fstring_expr
+
+            def fstring_single(node):
+                f_conversion(node)
+                self.default(node)
+            self.n_fstring_single = fstring_single
+
+            # def kwargs_only_36(node):
+            #     keys = node[-1].attr
+            #     num_kwargs = len(keys)
+            #     values = node[:num_kwargs]
+            #     for i, (key, value) in enumerate(zip(keys, values)):
+            #         self.write(key + '=')
+            #         self.preorder(value)
+            #         if i < num_kwargs:
+            #             self.write(',')
+            #     self.prune()
+            #     return
+            # self.n_kwargs_only_36 = kwargs_only_36
+
+            def n_call_kw36(node):
+                self.template_engine(("%c(", 0), node)
+                keys = node[-2].attr
+                num_kwargs = len(keys)
+                num_posargs = len(node) - (num_kwargs + 2)
+                n = len(node)
+                assert n >= len(keys)+1, \
+                  'not enough parameters keyword-tuple values'
+                sep = ''
+
+                line_number = self.line_number
+                for i in range(1, num_posargs):
+                    self.write(sep)
+                    self.preorder(node[i])
+                    if line_number != self.line_number:
+                        sep = ",\n" + self.indent + "  "
+                    else:
+                        sep = ", "
+                    line_number = self.line_number
+
+                i = num_posargs
+                j = 0
+                # FIXME: adjust output for line breaks?
+                while i < n-2:
+                    self.write(sep)
+                    self.write(keys[j] + '=')
+                    self.preorder(node[i])
+                    if line_number != self.line_number:
+                        sep = ",\n" + self.indent + "  "
+                    else:
+                        sep = ", "
+                    i += 1
+                    j += 1
+                self.write(')')
+                self.prune()
+                return
+            self.n_call_kw36 = n_call_kw36
+
+            def starred(node):
+                l = len(node)
+                assert l > 0
+                pos_args = node[0]
+                if pos_args == 'expr':
+                    pos_args = pos_args[0]
+                if pos_args == 'tuple':
+                    build_tuple = pos_args[0]
+                    if build_tuple.kind.startswith('BUILD_TUPLE'):
+                        tuple_len = 0
+                    else:
+                        tuple_len = len(node) - 1
+                    star_start = 1
+                    template = '%C', (0, -1, ', ')
+                    self.template_engine(template, pos_args)
+                    if tuple_len == 0:
+                        self.write("*()")
+                        # That's it
+                        self.prune()
+                    self.write(', ')
+                else:
+                    star_start = 0
+                if l > 1:
+                    template = ( '*%C', (star_start, -1, ', *') )
+                else:
+                    template = ( '*%c', (star_start, 'expr') )
+
+                self.template_engine(template, node)
+                self.prune()
+
+            self.n_starred = starred
+
+            def return_closure(node):
+                # Nothing should be output here
+                self.prune()
+                return
+            self.n_return_closure = return_closure
+            pass # version >= 3.6
+        pass # version >= 3.4
+    return

uncompyle6/semantics/make_function.py

@@ -506,6 +506,10 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         lambda_index = None
 
     args_node = node[-1]
+
+    # Get a list of tree nodes that constitute the values for the "default
+    # parameters"; these are default values that appear before any *, and are
+    # not to be confused with keyword parameters which may appear after *.
     if isinstance(args_node.attr, tuple):
         pos_args, kw_args, annotate_argc  = args_node.attr
         # FIXME: there is probably a better way to classify this.
@@ -516,13 +520,23 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
             lc_index = -3
             pass
 
-        if (self.version <= 3.3 and len(node) > 2 and
+        if (3.1 <= self.version <= 3.3 and len(node) > 2 and
                 node[lambda_index] != 'LOAD_LAMBDA' and
                 (have_kwargs or node[lc_index].kind != 'load_closure')):
+
+            # Find the index in "node" where the first default
+            # parameter value is located. Note this is in contrast to
+            # key-word arguments, pairs of (name, value), which appear after "*".
+            # "default_values_start" is this location.
+            default_values_start = 0
+            if node[0] == 'no_kwargs':
+                default_values_start += 1
             # args are after kwargs; kwargs are bundled as one node
-            defparams = node[1:args_node.attr[0]+1]
+            if node[default_values_start] == 'kwargs':
+                default_values_start += 1
+            defparams = node[default_values_start:default_values_start+args_node.attr[0]]
         else:
-            # args are before kwargs; kwags as bundled as one node
+            # args are first, before kwargs. Or there simply are no kwargs.
             defparams = node[:args_node.attr[0]]
             pass
     else:
@@ -581,7 +595,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
     paramnames = list(scanner_code.co_varnames[:argc])
 
     # defaults are for last n parameters, thus reverse
-    if not 3.0 <= self.version <= 3.1 or self.version >= 3.6:
+    if not 3.0 == self.version or self.version >= 3.6:
         paramnames.reverse(); defparams.reverse()
 
     try:
@@ -611,7 +625,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
     else:
         params = paramnames
 
-    if not 3.0 <= self.version <= 3.1 or self.version >= 3.6:
+    if not 3.0 == self.version or self.version >= 3.6:
         params.reverse() # back to correct order
 
         if code_has_star_arg(code):

uncompyle6/semantics/pysource.py

@@ -135,7 +135,7 @@ from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DE
 from uncompyle6.scanner import Code, get_scanner
 import uncompyle6.parser as python_parser
 from uncompyle6.semantics.make_function import (
-    make_function2, make_function3, make_function3_annotate,
+    make_function2, make_function3
     )
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.check_ast import checker
@@ -147,7 +147,7 @@ from uncompyle6.scanners.tok import Token
 from uncompyle6.semantics.consts import (
     LINE_LENGTH, RETURN_LOCALS, NONE, RETURN_NONE, PASS,
     ASSIGN_DOC_STRING, NAME_MODULE, TAB,
-    INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT, MAP_DIRECT,
+    INDENT_PER_LEVEL, TABLE_R, MAP_DIRECT,
     MAP, PRECEDENCE, ASSIGN_TUPLE_PARAM, escape, minint)
 
 
@@ -256,223 +256,6 @@ class SourceWalker(GenericASTTraversal, object):
             self.write("\n" + self.indent + INDENT_PER_LEVEL[:-1])
         return self.line_number
 
-    def customize_for_version(self, is_pypy, version):
-        if is_pypy:
-            ########################
-            # PyPy changes
-            #######################
-            TABLE_DIRECT.update({
-                'assert_pypy':	( '%|assert %c\n' , 1 ),
-                'assert2_pypy':	( '%|assert %c, %c\n' , 1, 4 ),
-                'try_except_pypy':	   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
-                'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ),
-                'assign3_pypy':        ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ),
-                'assign2_pypy':        ( '%|%c, %c = %c, %c\n', 3, 2, 0, 1),
-                })
-        else:
-            ########################
-            # Without PyPy
-            #######################
-            TABLE_DIRECT.update({
-                'assert':		( '%|assert %c\n' , 0 ),
-                'assert2':		( '%|assert %c, %c\n' , 0, 3 ),
-                'try_except':	( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
-                'assign2':      ( '%|%c, %c = %c, %c\n', 3, 4, 0, 1 ),
-                'assign3':      ( '%|%c, %c, %c = %c, %c, %c\n', 5, 6, 7, 0, 1, 2 ),
-                })
-        if version < 3.0:
-            TABLE_R.update({
-                'STORE_SLICE+0':	( '%c[:]', 0 ),
-                'STORE_SLICE+1':	( '%c[%p:]', 0, (1, 100) ),
-                'STORE_SLICE+2':	( '%c[:%p]', 0, (1, 100) ),
-                'STORE_SLICE+3':	( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
-                'DELETE_SLICE+0':	( '%|del %c[:]\n', 0 ),
-                'DELETE_SLICE+1':	( '%|del %c[%c:]\n', 0, 1 ),
-                'DELETE_SLICE+2':	( '%|del %c[:%c]\n', 0, 1 ),
-                'DELETE_SLICE+3':	( '%|del %c[%c:%c]\n', 0, 1, 2 ),
-            })
-            TABLE_DIRECT.update({
-                'raise_stmt2':	( '%|raise %c, %c\n', 0, 1),
-            })
-        else:
-            TABLE_DIRECT.update({
-                # Gotta love Python for its futzing around with syntax like this
-                'raise_stmt2':	 ( '%|raise %c from %c\n', 0, 1),
-            })
-
-            if version >= 3.2:
-                TABLE_DIRECT.update({
-                'del_deref_stmt': ( '%|del %c\n', 0),
-                'DELETE_DEREF': ( '%{pattr}', 0 ),
-                })
-
-        if version <= 2.4:
-            TABLE_DIRECT.update({
-                'importmultiple': ( '%|import %c%c\n', 2, 3),
-                'import_cont'   : ( ', %c', 2),
-                'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-',
-                                    (1, 'suite_stmts_opt') ,
-                                    (5, 'suite_stmts_opt') )
-                })
-            if version == 2.3:
-                TABLE_DIRECT.update({
-                    'if1_stmt':	( '%|if 1\n%+%c%-', 5 )
-                })
-
-            global NAME_MODULE
-            NAME_MODULE = AST('stmt',
-                              [ AST('assign',
-                                    [ AST('expr',
-                                          [Token('LOAD_GLOBAL', pattr='__name__',
-                                                 offset=0, has_arg=True)]),
-                                      AST('store',
-                                          [ Token('STORE_NAME', pattr='__module__',
-                                                  offset=3, has_arg=True)])
-                                    ])])
-            pass
-            if version <= 2.3:
-                if version <= 2.1:
-                    TABLE_DIRECT.update({
-                        'importmultiple': ( '%c', 2 ),
-                        # FIXME: not quite right. We have indiividual imports
-                        # when there is in fact one: "import a, b, ..."
-                        'imports_cont': ( '%C%,', (1, 100, '\n') ),
-                    })
-                    pass
-                pass
-            pass
-        elif version >= 2.5:
-            ########################
-            # Import style for 2.5+
-            ########################
-            TABLE_DIRECT.update({
-                'importmultiple': ( '%|import %c%c\n', 2, 3 ),
-                'import_cont'   : ( ', %c', 2 ),
-                # With/as is allowed as "from future" thing in 2.5
-                # Note: It is safe to put the variables after "as" in parenthesis,
-                # and sometimes it is needed.
-                'withstmt':     ( '%|with %c:\n%+%c%-', 0, 3),
-                'withasstmt':   ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
-            })
-
-            # In 2.5+ "except" handlers and the "finally" can appear in one
-            # "try" statement. So the below has the effect of combining the
-            # "tryfinally" with statement with the "try_except" statement
-            def tryfinallystmt(node):
-                if len(node[1][0]) == 1 and node[1][0][0] == 'stmt':
-                    if node[1][0][0][0] == 'try_except':
-                        node[1][0][0][0].kind = 'tf_try_except'
-                    if node[1][0][0][0] == 'tryelsestmt':
-                        node[1][0][0][0].kind = 'tf_tryelsestmt'
-                self.default(node)
-            self.n_tryfinallystmt = tryfinallystmt
-
-        ########################################
-        # Python 2.6+
-        #    except <condition> as <var>
-        # vs. older:
-        #    except <condition> , <var>
-        #
-        # For 2.6 we use the older syntax which
-        # matches how we parse this in bytecode
-        ########################################
-        if version > 2.6:
-            TABLE_DIRECT.update({
-                'except_cond2':	( '%|except %c as %c:\n', 1, 5 ),
-            })
-        else:
-            TABLE_DIRECT.update({
-                'except_cond3':	 ( '%|except %c, %c:\n', 1, 6 ),
-                'testtrue_then': ( 'not %p', (0, 22) ),
-
-            })
-
-        if 2.4 <= version <= 2.6:
-            TABLE_DIRECT.update({
-                'comp_for':	( ' for %c in %c', 3, 1 ),
-            })
-        else:
-            TABLE_DIRECT.update({
-                'comp_for':	( ' for %c in %c%c', 2, 0, 3 ),
-            })
-
-        if  version >= 3.0:
-            TABLE_DIRECT.update({
-                'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
-                'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
-                })
-
-            def n_mkfunc_annotate(node):
-
-                if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
-                    # LOAD_CONST code object ..
-                    # LOAD_CONST        'x0'  if >= 3.3
-                    # EXTENDED_ARG
-                    # MAKE_FUNCTION ..
-                    code = node[-4]
-                elif node[-3] == 'expr':
-                    code = node[-3][0]
-                else:
-                    # LOAD_CONST code object ..
-                    # MAKE_FUNCTION ..
-                    code = node[-3]
-
-                self.indent_more()
-                for annotate_last in range(len(node)-1, -1, -1):
-                    if node[annotate_last] == 'annotate_tuple':
-                        break
-
-                # FIXME: the real situation is that when derived from
-                # function_def_annotate we the name has been filled in.
-                # But when derived from funcdefdeco it hasn't Would like a better
-                # way to distinquish.
-                if self.f.getvalue()[-4:] == 'def ':
-                    self.write(code.attr.co_name)
-
-                # FIXME: handle and pass full annotate args
-                make_function3_annotate(self, node, is_lambda=False,
-                                        codeNode=code, annotate_last=annotate_last)
-
-                if len(self.param_stack) > 1:
-                    self.write('\n\n')
-                else:
-                    self.write('\n\n\n')
-                self.indent_less()
-                self.prune() # stop recursing
-            self.n_mkfunc_annotate = n_mkfunc_annotate
-
-            if version >= 3.4:
-                ########################
-                # Python 3.4+ Additions
-                #######################
-                TABLE_DIRECT.update({
-                    'LOAD_CLASSDEREF':	( '%{pattr}', ),
-                    })
-                ########################
-                # Python 3.5+ Additions
-                #######################
-                if version >= 3.5:
-                    TABLE_DIRECT.update({
-                        'await_expr':	       ( 'await %c', 0),
-                        'await_stmt':	       ( '%|%c\n', 0),
-                        'async_for_stmt':      (
-                            '%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
-                        'async_forelse_stmt':  (
-                            '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
-                        'async_with_stmt':     (
-                            '%|async with %c:\n%+%c%-', 0, 7),
-                        'async_with_as_stmt':  (
-                            '%|async with %c as %c:\n%+%c%-', 0, 6, 7),
-                        'unmap_dict':	       ( '{**%C}', (0, -1, ', **') ),
-                        # 'unmapexpr':	       ( '{**%c}', 0), # done by n_unmapexpr
-
-                    })
-
-
-                pass # version >= 3.4
-            pass # version >= 3.0
-        return
-
     f = property(lambda s: s.params['f'],
                  lambda s, x: s.params.__setitem__('f', x),
                  lambda s: s.params.__delitem__('f'),
@@ -624,26 +407,14 @@ class SourceWalker(GenericASTTraversal, object):
             self.prune() # stop recursing
 
     def n_yield(self, node):
-        self.write('yield')
         if node != AST('yield', [NONE, Token('YIELD_VALUE')]):
-            self.write(' ')
-            self.preorder(node[0])
+            self.template_engine(( 'yield %c', 0), node)
         elif self.version <= 2.4:
             # Early versions of Python don't allow a plain "yield"
-            self.write(' None')
-
-        self.prune() # stop recursing
-
-    # In Python 3.3+ only
-    def n_yield_from(self, node):
-        self.write('yield from')
-        self.write(' ')
-        if 3.3 <= self.version <= 3.4:
-            self.preorder(node[0][0][0][0])
-        elif self.version >= 3.5:
-            self.preorder(node[0])
+            self.write('yield None')
         else:
-            assert False, "dunno about this python version"
+            self.write('yield')
+
         self.prune() # stop recursing
 
     def n_build_slice3(self, node):
@@ -1622,9 +1393,15 @@ class SourceWalker(GenericASTTraversal, object):
     def print_super_classes3(self, node):
         n = len(node) - 1
         if node.kind != 'expr':
-            assert node[n].kind.startswith('CALL_FUNCTION')
+            if node == 'kwarg':
+                self.write('(')
+                self.template_engine(('%[0]{pattr}=%c', 1), node)
+                self.write(')')
+                return
 
             kwargs = None
+            assert node[n].kind.startswith('CALL_FUNCTION')
+
             if node[n].kind.startswith('CALL_FUNCTION_KW'):
                 # 3.6+ starts does this
                 kwargs = node[n-1].attr
@@ -1632,12 +1409,13 @@ class SourceWalker(GenericASTTraversal, object):
                 i = n - (len(kwargs)+1)
                 j = 1 + n - node[n].attr
             else:
-                for i in range(n-2, 0, -1):
-                    if not node[i].kind in ['expr', 'LOAD_CLASSNAME']:
+                start = n-2
+                for i in range(start, 0, -1):
+                    if not node[i].kind in ['expr', 'call', 'LOAD_CLASSNAME']:
                         break
                     pass
 
-                if i == n-2:
+                if i == start:
                     return
                 i += 2
 
@@ -1905,6 +1683,7 @@ class SourceWalker(GenericASTTraversal, object):
             else:
                 self.write('('); endchar = ')'
                 pass
+
         elif lastnodetype.startswith('BUILD_SET'):
             self.write('{'); endchar = '}'
         elif lastnodetype.startswith('BUILD_MAP_UNPACK'):

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.1.1'
+VERSION='3.1.2'