Merge branch 'master' into python-2.4

bugs still remain, just reduced.

NEWS

@@ -1,3 +1,16 @@
+uncompyle6 3.1.1 2018-04-01 Easter April Fool's
+
+Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
+
+- fill out 3.5+ BUILD_MAP_UNPACK (more work is needed)
+- fill out 3.4+ CALL_FUNCTION_... (more work is needed)
+- fill out 3.5 MAKE_FUNCTION  (more work is needed)
+- reduce 3.5, 3.6 control-flow bugs
+- reduce ambiguity in rules that lead to long (exponential?) parses
+- limit/isolate some 2.6/2.7,3.x grammar rules
+- more runtime testing of decompiled code
+- more removal of parenthesis around calls via setting precidence
+
 uncompyle6 3.1.0 2018-03-21 Equinox
 
 - Add code_deparse_with_offset() fragment function.

pytest/test_grammar.py

@@ -18,27 +18,28 @@ def test_grammar():
      right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['expr1024', 'pos_arg', 'get_iter', 'attribute'])
+    expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
-    unused_rhs = set(['list', 'mkfunc',
+    unused_rhs = set(['list', 'mkfunc', 'dict',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
+    expect_lhs.add('kvlist')
+    expect_lhs.add('kv3')
     if PYTHON3:
         expect_lhs.add('load_genexpr')
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
 
         unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp classdefdeco2
-        dict
+        except_pop_except generator_exp
         """.split()))
         if PYTHON_VERSION >= 3.0:
             expect_lhs.add("annotate_arg")
             expect_lhs.add("annotate_tuple")
             unused_rhs.add("mkfunc_annotate")
             unused_rhs.add('call')
+            unused_rhs.add("dict_comp")
+            unused_rhs.add("classdefdeco1")
             if PYTHON_VERSION < 3.6:
                 # 3.6 has at least one non-custom call rule
                 # the others don't
@@ -51,7 +52,6 @@ def test_grammar():
             else:
                 expect_right_recursive.add((('l_stmts',
                                              ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                # expect_lhs.add('kwargs1')
                 pass
             pass
         pass

test/.gitignore

@@ -0,0 +1 @@
+/nohup.out

test/Makefile

@@ -3,9 +3,9 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-5.6 5.6 5.8 \
-      grammar-coverage-2.5 grammar-coverage-2.6 grammarcoverage-2.7 \
-      grammar-coverage-3.1 grammar-coverage-3.2 grammarcoverage-3.3 \
-      grammar-coverage-3.4 grammar-coverage-3.5 grammarcoverage-3.6
+      grammar-coverage-2.5 grammar-coverage-2.6 grammar-coverage-2.7 \
+      grammar-coverage-3.1 grammar-coverage-3.2 grammar-coverage-3.3 \
+      grammar-coverage-3.4 grammar-coverage-3.5 grammar-coverage-3.6
 
 
 GIT2CL ?= git2cl
@@ -41,14 +41,17 @@ check-3.1: check-bytecode
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
 	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
@@ -117,26 +120,26 @@ check-bytecode-2.5:
 #: Get grammar coverage for Python 2.4
 grammar-coverage-2.4:
 	-rm $(COVER_DIR)/spark-grammar-24.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pyenvlib.py --2.4.6 --max= 800
 
 #: Get grammar coverage for Python 2.5
 grammar-coverage-2.5:
-	-rm $(COVER_DIR)/spark-grammar-25.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+	-rm $(COVER_DIR)/spark-grammar-2.5.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pyenvlib.py --2.5.6 --max=800
 
 #: Get grammar coverage for Python 2.6
 grammar-coverage-2.6:
-	-rm $(COVER_DIR)/spark-grammar-26.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+	-rm $(COVER_DIR)/spark-grammar-2.6.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pyenvlib.py --2.6.9 --max=800
 
 #: Get grammar coverage for Python 2.7
 grammar-coverage-2.7:
-	-rm $(COVER_DIR)/spark-grammar-27.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -147,33 +150,39 @@ SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-30.cover $(PYTHON) test_pythonl
 
 #: Get grammar coverage for Python 3.1
 grammar-coverage-3.1:
-	-rm $(COVER_DIR)/spark-grammar-31.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pyenvlib.py --3.1.5
+	-rm $(COVER_DIR)/spark-grammar-3.1.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pyenvlib.py --3.1.5
 
 #: Get grammar coverage for Python 3.2
 grammar-coverage-3.2:
-	-rm $(COVER_DIR)/spark-grammar-32.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
+	-rm $(COVER_DIR)/spark-grammar-3.2.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pyenvlib.py --3.2.6
 
 #: Get grammar coverage for Python 3.3
 grammar-coverage-3.3:
-	-rm $(COVER_DIR)/spark-grammar-33.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pyenvlib.py --3.3.6
+	-rm $(COVER_DIR)/spark-grammar-3.3.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pyenvlib.py --3.3.7 --max=800
 
 #: Get grammar coverage for Python 3.4
 grammar-coverage-3.4:
-	-rm $(COVER_DIR)/spark-grammar-34.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2
+	-rm $(COVER_DIR)/spark-grammar-3.4.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pyenvlib.py --3.4.8 --max=800
 
 #: Get grammar coverage for Python 3.5
 grammar-coverage-3.5:
-	rm $(COVER_DIR)/spark-grammar-35.cover || /bin/true
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pyenvlib.py --3.5.3
+	rm $(COVER_DIR)/spark-grammar-3.5.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pyenvlib.py --3.5.5 --max=450
+
+#: Get grammar coverage for Python 3.6
+grammar-coverage-3.6:
+	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
@@ -196,6 +205,7 @@ check-bytecode-3.1:
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:

test/grammar-cover/.gitignore

@@ -0,0 +1 @@
+/.python-version

test/grammar-cover/README.md

@@ -0,0 +1 @@
+Code in this directory gets statistics on grammar coverage

test/grammar-cover/convert.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+for VERS in 2{4,5,6,7} 3{2,3,4,5} ; do
+    GRAMMAR_TXT=grammar-${VERS}.txt
+    spark-parser-coverage --max-count 3000 --path spark-grammar-${VERS}.cover > $GRAMMAR_TXT
+done

test/grammar-cover/grammar-all.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+$SHELL ./grammar.sh 2.4 2.5 2.6 2.7 3.2 3.3 3.4 3.5 3.6

test/grammar-cover/grammar.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Remake Python grammar statistics
+
+typeset -A ALL_VERS=([2.4]=2.4.6 [2.5]=2.5.6 [2.6]=2.6.9 [2.7]=2.7.14 [3.2]=3.2.6 [3.3]=3.3.6 [3.4]=3.4.8 [3.5]=3.5.5 [3.6]=3.6.4)
+
+if (( $# == 0 )); then
+    echo 1>&2 "usage: $0 two-digit-version"
+    exit 1
+fi
+
+me=${BASH_SOURCE[0]}
+workdir=$(dirname $me)
+cd $workdir
+workdir=$(pwd)
+while [[ -n $1 ]]  ; do
+    SHORT_VERSION=$1; shift
+    LONG_VERSION=${ALL_VERS[$SHORT_VERSION]}
+    if [[ -z ${LONG_VERSION} ]] ; then
+       echo 1>&2 "Version $SHORT_VERSION not known"
+       exit 2
+    fi
+
+    tmpdir=$workdir/../../tmp/grammar-cover
+    COVER_FILE=${tmpdir}/spark-grammar-${SHORT_VERSION}.cover
+    [[ -d  $tmpdir ]] || mkdir $tmpdir
+    cd $workdir/../..
+    if [[ $SHORT_VERSION > 2.5 ]] ; then
+        source ./admin-tools/setup-master.sh
+    else
+        source ./admin-tools/setup-python-2.4.sh
+    fi
+    GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
+    pyenv local ${LONG_VERSION}
+    cd ./test
+    if [[ -r $COVER_FILE ]]; then
+	rm $COVER_FILE
+    fi
+    if [[ -r $GRAMMAR_TXT ]]; then
+        GRAMMAR_SAVE_TXT=${tmpdir}/grammar-${SHORT_VERSION}-save.txt
+        cp $GRAMMAR_TXT $GRAMMAR_SAVE_TXT
+    fi
+    make grammar-coverage-${SHORT_VERSION};
+    spark-parser-coverage --max-count=3000 --path $COVER_FILE > $GRAMMAR_TXT
+done

test/grammar-cover/run-and-email.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+USER=${USER:-rocky}
+EMAIL=${EMAIL:-rb@dustyfeet.com}
+SUBJECT_PREFIX="grammar cover testing for"
+LOGFILE=/tmp/grammar-cover-$$.log
+/bin/bash ./grammar-all.sh  >$LOGFILE 2>&1
+rc=$?
+if ((rc == 0)); then
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX ok" ${USER}@localhost
+else
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX not ok" ${USER}@localhost
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX not ok" $EMAIL
+fi

test/simple_source/bug25/04_try_except_else.py

@@ -0,0 +1,18 @@
+# Bug found in 2.4 test_math.py
+# Bug was turning last try/except/else into try/else
+import math
+def test_exceptions():
+    try:
+        x = math.exp(-1000000000)
+    except:
+        raise RuntimeError
+
+    x = 1
+    try:
+        x = math.sqrt(-1.0)
+    except ValueError:
+        return x
+    else:
+        raise RuntimeError
+
+test_exceptions()

test/simple_source/bug27+/02_assert.py

@@ -1,5 +1,5 @@
 # From 2.7 test_argparse.py
-# Bug was turnning assert into an "or raise" statement
+# Bug was turning assert into an "or raise" statement
 def __call__(arg, dest):
     try:
         assert arg == 'spam', 'dest: %s' % dest
@@ -15,3 +15,17 @@ def refactor_doctest(clipped, new):
     if not new:
         new += u"\n"
     return
+
+# From 2.7.14 test_hashlib.py
+# The bug was turning assert into an "if"
+# statement which isn't wrong, but we got the
+# range of the if incorrect. When we have
+# better control flow analysis we can revisit.
+def test_threaded_hashing():
+    for threadnum in xrange(1):
+        result = 1
+        assert result > 0
+        result = 2
+    return result
+
+assert test_threaded_hashing() == 2

test/simple_source/bug27+/05_not_unconditional.py

@@ -1,5 +1,5 @@
 # From 2.7 test_itertools.py
-# Bug was in 2.7 decompiling like the commented out
+# Bug was in 2.7 decompiling the target assignment
 # code below
 from itertools import izip_longest
 for args in [

test/simple_source/bug35/01_loop_if_continue.py

@@ -0,0 +1,13 @@
+# From 3.6.4 pathlib.py
+# Bug was handling "continue" as last statement of "if"
+# RUNNABLE!
+def parse_parts(it, parts):
+    for part in it:
+        if not part:
+            continue
+        parts = 1
+    return parts
+
+assert parse_parts([], 5) == 5
+assert parse_parts([True], 6) == 1
+assert parse_parts([False], 6) == 6

test/simple_source/bug35/01_map_unpack.py

@@ -1,9 +1,19 @@
 # Python 3.5+ PEP 448 - Additional Unpacking Generalizations for dictionaries
-{**{}}
-{**{'a': 1, 'b': 2}}
-## {**{'x': 1}, **{'y': 2}}
+# RUNNABLE!
+b = {**{}}
+assert b == {}
+c = {**{'a': 1, 'b': 2}}
+assert c == {'a': 1, 'b': 2}
+d = {**{'x': 1}, **{'y': 2}}
+assert d == {'x': 1, 'y': 2}
 # {'c': 1, {'d': 2}, **{'e': 3}}
 [*[]]
-{**{0:0 for a in b}}
-## {**{}, **{}}
-## {**{}, **{}, **{}}
+
+assert {0: 0} == {**{0:0 for a in c}}
+
+# FIXME: assert deparsing is incorrect for:
+# {**{}, **{}}
+# assert {} == {**{}, **{}, **{}}
+
+# {**{}, **{}, **{}}
+# assert {} == {**{}, **{}, **{}}

test/simple_source/bug35/01_while_if_then.py

@@ -0,0 +1,23 @@
+# From python 3.5.5 telnetlib
+# The bug is the end of a "then" jumping
+# back to the loop which could look like
+# a "continue" and also not like a then/else
+# break
+def process_rawq(self, cmd, cmd2):
+    while self.rawq:
+        if self.iacseq:
+            if cmd:
+                pass
+            elif cmd2:
+                if self.option_callback:
+                    self.option = 2
+                else:
+                    self.option = 3
+
+# From python 3.5.5 telnetlib
+def listener(data):
+    while 1:
+        if data:
+            data = 1
+        else:
+            data = 2

test/simple_source/bug35/04_call_function.py

@@ -1,6 +1,8 @@
 # From sql/schema.py and 3.5 _strptime.py
 # Note that kwargs comes before "positional" args
 
+# RUNNABLE!
+
 def tometadata(self, metadata, schema, Table, args, name=None):
     table = Table(
         name, metadata, schema=schema,
@@ -10,3 +12,49 @@ def tometadata(self, metadata, schema, Table, args, name=None):
 
 def _strptime_datetime(cls, args):
     return cls(*args)
+
+
+# From 3.5.5 imaplib
+# Bug is in parsing *date_time[:6] parameter
+from datetime import datetime, timezone, timedelta
+import time
+def Time2Internaldate(date_time):
+    delta = timedelta(seconds=0)
+    return datetime(*date_time[:6], tzinfo=timezone(delta))
+
+assert Time2Internaldate(time.localtime())
+
+# From 3.5.5 tkinter/dialog.py
+def test_varargs0_ext():
+    try:
+        {}.__contains__(*())
+    except TypeError:
+        pass
+
+test_varargs0_ext()
+
+# From 3.4.6 tkinter/dialog.py
+# Bug is in position of *cnf.
+
+def __init__(self, cnf={}):
+    self.num = self.tk.call(
+        'tk_dialog', self._w,
+        cnf['title'], cnf['text'],
+        cnf['bitmap'], cnf['default'],
+        *cnf['strings'])
+
+# From python 3.4.8 multiprocessing/context.py
+def Value(self, fn, typecode_or_type, *args, lock=True):
+    return fn(typecode_or_type, *args, lock=lock,
+              ctx=self.get_context())
+
+# From 3.6.4 heapq.py
+def merge(*iterables, key=None, reverse=False):
+    return
+
+def __call__(self, *args, **kwds):
+    pass
+
+# From 3.6.4 shutil
+def unpack_archive(func, filename, dict, format_info, extract_dir=None):
+    func(filename, extract_dir, **dict(format_info[2]))

test/simple_source/bug35/06_try_return.py

@@ -0,0 +1,12 @@
+# From 3.6.4 pdb.py
+# Bug was not having a semantic action for "except_return" tree
+def do_commands(self, arg):
+    if not arg:
+        bnum = 1
+    else:
+        try:
+            bnum = int(arg)
+        except:
+            self.error("Usage:")
+            return
+    self.commands_bnum = bnum

test/simple_source/bug36/04_try_finally.py

@@ -31,3 +31,18 @@ def handle_read(self):
         return why
 
     return data
+
+# From 3.6 contextlib
+# Bug is indentation of "return exc"
+# Also there are extra statements to remove exec,
+# which we hide (unless doing fragments).
+# Note: The indentation bug may be a result of using improper
+# grammar.
+def __exit__(self, type, value, traceback):
+    try:
+        value()
+    except StopIteration as exc:
+        return exc
+    except RuntimeError as exc:
+        return exc
+    return

test/simple_source/bug36/05-for-ifelse.py

@@ -0,0 +1,13 @@
+# From 3.6.4 configparser.py
+# Bug in 3.6 was handling "else" with compound
+# if. there is no POP_BLOCK and
+# there are several COME_FROMs before the else
+def _read(self, fp, a, value, f):
+    for line in fp:
+        for prefix in a:
+            fp()
+        if (value and fp and
+            prefix > 5):
+            f()
+        else:
+            f()

test/simple_source/bug36/10_argparse.py

@@ -0,0 +1,18 @@
+# From 3.6.4 test_argparse.py
+# Bug was in parsing ** args
+import argparse
+def test_namespace_starkwargs_notidentifier(self):
+    ns = argparse.Namespace(**{'"': 'quote'})
+    string = """Namespace(**{'"': 'quote'})"""
+    assert ns ==  string
+
+def test_namespace_kwargs_and_starkwargs_notidentifier(self):
+    ns = argparse.Namespace(a=1, **{'"': 'quote'})
+    string = """Namespace(a=1, **{'"': 'quote'})"""
+    assert ns == string
+
+
+def test_namespace(self):
+    ns = argparse.Namespace(foo=42, bar='spam')
+    string = "Namespace(bar='spam', foo=42)"
+    assert ns == string

test/simple_source/exception/04_assert_continue.py

@@ -0,0 +1,16 @@
+# Adapted from Python 3.3 idlelib/PyParse.py
+# Bug is continue flowing back to while messing up the determination
+# that it is inside an "if".
+
+# RUNNABLE!
+def _study1(i, n, ch):
+    while i == 3:
+        i = 4
+        if ch:
+            i = 10
+            assert i < 5
+            continue
+        if n:
+            return n
+
+assert _study1(3, 4, False) == 4

test/simple_source/stmts/15_assert.py

@@ -1,3 +1,5 @@
+# RUNNABLE!
+
 # Tests:
 #   2.7:
 #   assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
@@ -16,7 +18,7 @@ for method_name in ['a']:
     if method_name in ('b',):
         method = 'a'
     else:
-        assert 0, "instance installed"
+        assert True, "instance installed"
 
     methods = 'b'
 
@@ -25,5 +27,9 @@ for method_name in ['a']:
 #   if not not do_setlocal:
 #      raise AssertError
 
+# Hmmm.. this isn't strickly a bug
+
 def getpreferredencoding(do_setlocale=True):
     assert not do_setlocale
+
+getpreferredencoding(False)

test/stdlib/.gitignore

@@ -0,0 +1 @@
+/.python-version

test/stdlib/run-and-email.sh

@@ -2,7 +2,7 @@
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
 SUBJECT_PREFIX="stdlib unit testing for"
-for VERSION in 2.7.14 2.6.9 ; do
+for VERSION in 3.4.8 3.5.5 3.6.4 ; do
     typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
     if ! pyenv local $VERSION ; then

test/stdlib/runtests.sh

@@ -78,7 +78,8 @@ case $PYVERSION in
 	    [test_curses.py]=1  # Possibly fails on its own but not detected
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_doctest.py]=1 # Fails on its own
-	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
+	    [test_generators.py]=1  # control flow. uncompyle2 has problem here too
+	    [test_grammar.py]=1     # Too many stmts. Handle large stmts
 	    [test_io.py]=1 # Test takes too long to run
 	    [test_ioctl.py]=1 # Test takes too long to run
 	    [test_itertools.py]=1 # Fix erroneous reduction to "conditional_true".
@@ -107,6 +108,7 @@ case $PYVERSION in
 	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation
 	    [test_decorators.py]=1  # Control flow wrt "if elif"
 	    [test_pow.py]=1         # Control flow wrt "continue"
+	    [test_quopri.py]=1      # Only fails on POWER
 	)
 	;;
     *)

uncompyle6/main.py

@@ -176,6 +176,7 @@ def main(in_base, out_base, files, codes, outfile=None,
 
     for filename in files:
         infile = os.path.join(in_base, filename)
+        # print("XXX", infile)
         if not os.path.exists(infile):
             sys.stderr.write("File '%s' doesn't exist. Skipped\n"
                              % infile)

uncompyle6/parser.py

@@ -565,9 +565,6 @@ class PythonParser(GenericASTBuilder):
 
         # Positional arguments in make_function
         pos_arg ::= expr
-
-        expr32 ::= expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr
-        expr1024 ::= expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32
         '''
 
     def p_store(self, args):

uncompyle6/parsers/parse2.py

@@ -107,11 +107,8 @@ class Python2Parser(PythonParser):
         _mklambda ::= load_closure mklambda
         kwarg     ::= LOAD_CONST expr
 
-        kvlist ::= kvlist kv3
         kv3 ::= expr expr STORE_MAP
 
-        dict ::= BUILD_MAP kvlist
-
         classdef ::= buildclass store
 
         buildclass ::= LOAD_CONST expr mkfunc
@@ -296,19 +293,8 @@ class Python2Parser(PythonParser):
             # The order of opname listed is roughly sorted below
             if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
                 v = token.attr
-                thousands = (v//1024)
-                thirty32s = ((v//32) % 32)
-                if thirty32s > 0:
-                    rule = "expr32 ::=%s" % (' expr' * 32)
-                    self.add_unique_rule(rule, opname_base, v, customize)
-                    self.seen32 = True
-                if thousands > 0:
-                    self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),
-                                         opname_base, v, customize)
-                    self.seen1024 = True
                 collection = opname_base[opname_base.find('_')+1:].lower()
-                rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +
-                        'expr32 '*thirty32s + 'expr '*(v % 32) + opname)
+                rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname)
                 self.add_unique_rules([
                     "expr ::= %s" % collection,
                     rule], customize)
@@ -328,11 +314,9 @@ class Python2Parser(PythonParser):
                             'dict_comp_func', 0, customize)
 
                 else:
-                    kvlist_n = "kvlist_%s" % token.attr
-                    self.add_unique_rules([
-                        (kvlist_n + " ::=" + ' kv3' * token.attr),
-                        "dict ::= %s %s" % (opname, kvlist_n)
-                    ], customize)
+                    kvlist_n =  ' kv3' * token.attr
+                    rule = "dict ::= %s%s" % (opname, kvlist_n)
+                    self.addRule(rule, nop_func)
                 continue
             elif opname_base == 'BUILD_SLICE':
                 slice_num  = token.attr
@@ -518,7 +502,7 @@ class Python2Parser(PythonParser):
             self.addRule(rule, nop_func)
             pass
 
-        self.check_reduce['aug_assign1'] = 'AST'
+        self.check_reduce['raise_stmt1'] = 'tokens'
         self.check_reduce['aug_assign2'] = 'AST'
         self.check_reduce['or'] = 'AST'
         # self.check_reduce['_stmts'] = 'AST'
@@ -538,7 +522,12 @@ class Python2Parser(PythonParser):
 
         if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
             return True
-        if rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
+        elif lhs in ('raise_stmt1',):
+            # We will assme 'LOAD_ASSERT' will be handled by an assert grammar rule
+            return (tokens[first] == 'LOAD_ASSERT' and
+                    (last >= len(tokens) or tokens[last] not in
+                     ('COME_FROM', 'JUMP_BACK','JUMP_FORWARD')))
+        elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
             expr2 = ast[2]
             return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
         return False

uncompyle6/parsers/parse25.py

@@ -73,7 +73,6 @@ class Python25Parser(Python26Parser):
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
         classdefdeco2 ::= LOAD_CONST expr mkfunc CALL_FUNCTION_0 BUILD_CLASS
         kv3 ::= expr expr STORE_MAP
-        kvlist ::= kvlist kv3
         ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
         return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
         return_if_stmt   ::= ret_expr RETURN_END_IF POP_TOP

uncompyle6/parsers/parse26.py

@@ -261,6 +261,9 @@ class Python26Parser(Python2Parser):
 
     def p_misc26(self, args):
         """
+        dict ::= BUILD_MAP kvlist
+        kvlist ::= kvlist kv3
+
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and          ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
 

uncompyle6/parsers/parse27.py

@@ -38,8 +38,6 @@ class Python27Parser(Python2Parser):
         comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
         comp_iter     ::= comp_if
-        comp_iter     ::= comp_if_not
-        comp_if_not   ::= expr jmp_true comp_iter
         comp_iter     ::= comp_body
 
         dict_comp_body ::= expr expr MAP_ADD

uncompyle6/parsers/parse3.py

@@ -84,8 +84,6 @@ class Python3Parser(PythonParser):
         stmt ::= dict_comp_func
         dict_comp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER store
                            comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
-        dict_comp      ::= LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_0 expr
-                           GET_ITER CALL_FUNCTION_1
 
         comp_iter     ::= comp_if
         comp_iter     ::= comp_if_not
@@ -113,9 +111,9 @@ class Python3Parser(PythonParser):
         continues ::= continue
 
 
-        kwarg   ::= LOAD_CONST expr
-        kwargs  ::= kwarg*
-        kwargs1 ::= kwarg+
+        kwarg      ::= LOAD_CONST expr
+        kwargs     ::= kwarg+
+
 
         classdef ::= build_class store
 
@@ -125,9 +123,8 @@ class Python3Parser(PythonParser):
 
         stmt ::= classdefdeco
         classdefdeco ::= classdefdeco1 store
-        classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
-        classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
 
+        expr   ::= LOAD_ASSERT
         assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
 
         assert_expr ::= expr
@@ -393,9 +390,6 @@ class Python3Parser(PythonParser):
         '''
         load_genexpr ::= LOAD_GENEXPR
         load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_CONST
-
-        # Is there something general going on here?
-        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_CLOSURE_0 expr GET_ITER CALL_FUNCTION_1
         '''
 
     def p_expr3(self, args):
@@ -511,7 +505,8 @@ class Python3Parser(PythonParser):
         self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
         if possible_class_decorator:
-            if next_token == 'CALL_FUNCTION' and next_token.attr == 1:
+            if (next_token == 'CALL_FUNCTION' and next_token.attr == 1
+                and args_pos > 1):
                 rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                         %  (('expr ' * (args_pos-1)), opname, args_pos))
                 self.add_unique_rule(rule, token.kind, uniq_param, customize)
@@ -531,7 +526,7 @@ class Python3Parser(PythonParser):
         subclassing is, well, is pretty base.  And we want it that way: lean and
         mean so that parsing will go faster.
 
-        Here, we add additional grammra rules based on specific instructions
+        Here, we add additional grammar rules based on specific instructions
         that are in the instruction/token stream. In classes that
         inherit from from here and other versions, grammar rules may
         also be removed.
@@ -575,6 +570,11 @@ class Python3Parser(PythonParser):
         seen_LOAD_BUILD_CLASS = False
         seen_GET_AWAITABLE_YIELD_FROM = False
 
+        # This is used in parse36.py as well as here
+        self.seen_LOAD_DICTCOMP = False
+        self.seen_LOAD_SETCOMP = False
+
+
         # Loop over instructions adding custom grammar rules based on
         # a specific instruction seen.
 
@@ -623,9 +623,7 @@ class Python3Parser(PythonParser):
                 self.addRule(rule, nop_func)
             elif opname.startswith('BUILD_LIST_UNPACK'):
                 v = token.attr
-                rule = ('build_list_unpack ::= ' + 'expr1024 ' * int(v//1024) +
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
+                rule = 'build_list_unpack ::= %s%s' % ('expr ' * v, opname)
                 self.addRule(rule, nop_func)
                 rule = 'expr ::= build_list_unpack'
                 self.addRule(rule, nop_func)
@@ -644,21 +642,31 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                     rule = "dict ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
-                    if opname != 'BUILD_MAP_WITH_CALL':
-                        if opname == 'BUILD_MAP_UNPACK':
+                    if not opname.startswith('BUILD_MAP_WITH_CALL'):
+                        # FIXME: Use the attr
+                        # so this doesn't run into exponential parsing time.
+                        if opname.startswith('BUILD_MAP_UNPACK'):
+                            # FIXME: start here
+                            # rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
                             rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
                             rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = 'dict ::= ' + 'dict_entry ' * token.attr
-                            self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = ('unmap_dict ::= ' +
-                                    ('dict ' * token.attr) +
-                                    'BUILD_MAP_UNPACK')
+                            rule = 'dict ::= %s' % ('dict_entry ' * token.attr)
+                            self.addRule(rule, nop_func)
+
+                            # FIXME: really we need a combination of dict_entry-like things.
+                            # It just so happens the most common case is not to mix
+                            # dictionary comphensions with dictionary, elements
+                            if self.seen_LOAD_DICTCOMP:
+                                rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)
+                                self.addRule(rule, nop_func)
+
+                            rule = 'unmap_dict ::= %s%s' % (('dict ' * token.attr), opname)
                         else:
-                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                            rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = "dict ::=  %s %s" % (kvlist_n, opname)
+                            rule = "dict ::=  %s" % kvlist_n
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
@@ -666,9 +674,7 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
                 v = token.attr
-                rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
+                rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)
                 self.addRule(rule, nop_func)
             elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
                 v = token.attr
@@ -691,9 +697,7 @@ class Python3Parser(PythonParser):
                         self.add_unique_rule(rule, opname, token.attr, customize)
                 if not is_LOAD_CLOSURE or v == 0:
                     collection = opname_base[opname_base.find('_')+1:].lower()
-                    rule = (('%s ::= ' % collection) + 'expr1024 ' * int(v//1024) +
-                            'expr32 ' * int((v//32) % 32) +
-                            'expr ' * (v % 32) + opname)
+                    rule = '%s ::= %s%s' % (collection, 'expr ' * v, opname)
                     self.add_unique_rules([
                         'expr ::= %s' % collection,
                         rule], customize)
@@ -716,6 +720,19 @@ class Python3Parser(PythonParser):
                                        'CALL_FUNCTION_VAR',
                                        'CALL_FUNCTION_VAR_KW'))
                   or opname.startswith('CALL_FUNCTION_KW')):
+
+                if opname == 'CALL_FUNCTION' and token.attr == 1:
+                    rule = """
+                     dict_comp    ::= LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_0 expr
+                                      GET_ITER CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    """
+                    if self.version < 3.5:
+                        rule += """
+                        classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                        """
+                    self.addRule(rule, nop_func)
+
                 self.custom_classfunc_rule(opname, token, customize,
                                            seen_LOAD_BUILD_CLASS,
                                            seen_GET_AWAITABLE_YIELD_FROM, tokens[i+1])
@@ -784,6 +801,7 @@ class Python3Parser(PythonParser):
                 self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
                 custom_ops_seen.add(opname)
             elif opname == 'LOAD_DICTCOMP':
+                self.seen_LOAD_DICTCOMP = True
                 if has_get_iter_call_function1:
                     rule_pat = ("dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
                                 "GET_ITER CALL_FUNCTION_1")
@@ -798,6 +816,7 @@ class Python3Parser(PythonParser):
             elif opname == 'LOAD_LISTCOMP':
                 self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
             elif opname == 'LOAD_SETCOMP':
+                self.seen_LOAD_SETCOMP = True
                 # Should this be generalized and put under MAKE_FUNCTION?
                 if has_get_iter_call_function1:
                     self.addRule("expr ::= set_comp", nop_func)
@@ -816,6 +835,17 @@ class Python3Parser(PythonParser):
                 # DRY with MAKE_FUNCTION
                 # Note: this probably doesn't handle kwargs proprerly
 
+                if opname == 'MAKE_CLOSURE_0' and self.seen_LOAD_DICTCOMP:
+                    # Is there something general going on here?
+                    # Note that 3.6+ doesn't do this, but we'll remove
+                    # this rule in parse36.py
+                    rule = """
+                        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
+                                      MAKE_CLOSURE_0 expr
+                                      GET_ITER CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
                 args_pos, args_kw, annotate_args  = token.attr
 
                 # FIXME: Fold test  into add_make_function_rule
@@ -854,13 +884,13 @@ class Python3Parser(PythonParser):
                                                  opname, token.attr, customize)
 
                 if args_kw > 0:
-                    kwargs_str = 'kwargs1 '
+                    kwargs_str = 'kwargs '
                 else:
                     kwargs_str = ''
 
                 # Note order of kwargs and pos args changed between 3.3-3.4
                 if self.version <= 3.2:
-                    rule = ('mkfunc ::= %s%sload_closure LOAD_CONST kwargs %s'
+                    rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'
                             % (kwargs_str, 'expr ' * args_pos, opname))
                 elif self.version == 3.3:
                     rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'
@@ -870,9 +900,11 @@ class Python3Parser(PythonParser):
                             % ('expr ' * args_pos, kwargs_str, opname))
 
                 self.add_unique_rule(rule, opname, token.attr, customize)
-                rule = ('mkfunc ::= %sload_closure load_genexpr %s'
-                        % ('pos_arg ' * args_pos, opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+
+                if args_kw == 0:
+                    rule = ('mkfunc ::= %sload_closure load_genexpr %s'
+                                % ('pos_arg ' * args_pos, opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
 
                 if self.version < 3.4:
                     rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'
@@ -968,30 +1000,41 @@ class Python3Parser(PythonParser):
                                 opname))
                     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
+                if args_kw == 0:
+                    kwargs = 'no_kwargs'
+                    self.add_unique_rule("no_kwargs ::=", opname, token.attr, customize)
+                else:
+                    kwargs = 'kwargs'
+
                 if self.version < 3.3:
                     # positional args after keyword args
-                    rule = ('mkfunc ::= kwargs %s%s %s' %
+                    rule = ('mkfunc ::= %s %s%s%s' %
+                            (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST ',
+                             opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    rule = ('mkfunc ::= %s%s%s' %
                             ('pos_arg ' * args_pos, 'LOAD_CONST ',
                              opname))
                 elif self.version == 3.3:
                     # positional args after keyword args
-                    rule = ('mkfunc ::= kwargs %s%s %s' %
-                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                    rule = ('mkfunc ::= %s %s%s%s' %
+                            (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST '*2,
                              opname))
                 elif self.version > 3.5:
                     # positional args before keyword args
-                    rule = ('mkfunc ::= %skwargs1 %s %s' %
-                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                    rule = ('mkfunc ::= %s%s %s%s' %
+                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
                              opname))
                 elif self.version > 3.3:
                     # positional args before keyword args
-                    rule = ('mkfunc ::= %skwargs %s %s' %
-                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                    rule = ('mkfunc ::= %s%s %s%s' %
+                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
                              opname))
                 else:
-                    rule = ('mkfunc ::= kwargs %sexpr %s' %
-                            ('pos_arg ' * args_pos, opname))
+                    rule = ('mkfunc ::= %s%sexpr %s' %
+                            (kwargs, 'pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+
                 if opname.startswith('MAKE_FUNCTION_A'):
                     if self.version >= 3.6:
                         rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %

uncompyle6/parsers/parse34.py

@@ -31,6 +31,9 @@ class Python34Parser(Python33Parser):
         expr ::= LOAD_ASSERT
 
 
+        # passtmt is needed for semantic actions to add "pass"
+        suite_stmts_opt ::= pass
+
         # Seems to be needed starting 3.4.4 or so
         while1stmt    ::= SETUP_LOOP l_stmts
                           COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP

uncompyle6/parsers/parse35.py

@@ -32,7 +32,7 @@ class Python35Parser(Python34Parser):
         #     ...
         # the end of the if will jump back to the loop and there will be a COME_FROM
         # after the jump
-        l_stmts ::= lastl_stmt COME_FROM l_stmts
+        l_stmts ::= lastl_stmt come_froms l_stmts
 
         # Python 3.5+ Await statement
         expr       ::= await_expr
@@ -101,7 +101,20 @@ class Python35Parser(Python34Parser):
 
         return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
 
+        jb_else     ::= JUMP_BACK ELSE
         ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
+        ifelsestmtl ::= testexpr c_stmts_opt jb_else else_suitel
+
+        # 3.5 Has jump optimization which can route the end of an
+        # "if/then" back to to a loop just before an else.
+        jump_absolute_else ::= jb_else
+        jump_absolute_else ::= CONTINUE ELSE
+
+        # Our hacky "ELSE" determination doesn't do a good job and really
+        # determine the start of an "else". It could also be the end of an
+        # "if-then" which ends in a "continue". Perhaps with real control-flow
+        # analysis we'll sort this out. Or call "ELSE" something more appropriate.
+        _ifstmts_jump ::= c_stmts_opt ELSE
 
         # ifstmt ::= testexpr c_stmts_opt
 
@@ -209,7 +222,6 @@ class Python35Parser(Python34Parser):
             self.add_unique_rule(rule, token.kind, uniq_param, customize)
             self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
 
-        uniq_param = args_kw + args_pos
         if opname.startswith('CALL_FUNCTION_VAR'):
             # Python 3.5 changes the stack position of *args. KW args come
             # after *args.
@@ -225,7 +237,10 @@ class Python35Parser(Python34Parser):
             rule = ('call ::= expr expr ' +
                     ('pos_arg ' * args_pos) +
                     ('kwarg ' * args_kw) + kw + token.kind)
-            self.add_unique_rule(rule, token.kind, uniq_param, customize)
+
+            # Note: semantic actions make use of the fact of wheter  "args_pos"
+            # zero or not in creating a template rule.
+            self.add_unique_rule(rule, token.kind, args_pos, customize)
         else:
             super(Python35Parser, self).custom_classfunc_rule(opname, token, customize,
                                                               seen_LOAD_BUILD_CLASS,

uncompyle6/parsers/parse36.py

@@ -35,9 +35,6 @@ class Python36Parser(Python35Parser):
         # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
         return_closure   ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
 
-        # Is there something general going on here? FIXME: Isolate to LOAD_DICTCOMP
-        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1
-
         stmt               ::= conditional_lambda
         conditional_lambda ::= expr jmp_false expr return_if_lambda
                                return_stmt_lambda LAMBDA_MARKER
@@ -51,8 +48,9 @@ class Python36Parser(Python35Parser):
         whilestmt       ::= SETUP_LOOP testexpr l_stmts_opt
                             JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
 
-        # This might be valid in < 3.6
+        # A COME_FROM is dropped off because of JUMP-to-JUMP optimization
         and  ::= expr jmp_false expr
+        and  ::= expr jmp_false expr jmp_false
 
         jf_cf       ::= JUMP_FORWARD COME_FROM
         conditional ::= expr jmp_false expr jf_cf expr COME_FROM
@@ -62,6 +60,9 @@ class Python36Parser(Python35Parser):
 
         except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
 
+        jb_cfs      ::= JUMP_BACK come_froms
+        ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel
+
         # In 3.6+, A sequence of statements ending in a RETURN can cause
         # JUMP_FORWARD END_FINALLY to be omitted from try middle
 
@@ -105,6 +106,23 @@ class Python36Parser(Python35Parser):
                     fstring_single ::= expr FORMAT_VALUE
                 """
                 self.add_unique_doc_rules(rules_str, customize)
+            elif opname == 'MAKE_FUNCTION_8':
+                if self.seen_LOAD_DICTCOMP:
+                    # Is there something general going on here?
+                    rule = """
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
+                                     MAKE_FUNCTION_8 expr
+                                     GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+                elif self.seen_LOAD_SETCOMP:
+                    rule = """
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_CONST
+                                    MAKE_FUNCTION_8 expr
+                                    GET_ITER CALL_FUNCTION_1
+                       """
+                    self.addRule(rule, nop_func)
+
             elif opname == 'BEFORE_ASYNC_WITH':
                 rules_str = """
                   stmt ::= async_with_stmt
@@ -195,11 +213,9 @@ class Python36Parser(Python35Parser):
             self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
 
         if opname.startswith('CALL_FUNCTION_KW'):
-            self.addRule("expr ::= call_kw", nop_func)
+            self.addRule("expr ::= call_kw36", nop_func)
             values = 'expr ' * token.attr
-            rule = 'call_kw ::= expr kwargs_36 %s' % token.kind
-            self.addRule(rule, nop_func)
-            rule = 'kwargs_36 ::= %s LOAD_CONST' % values
+            rule = "call_kw36 ::= expr %s LOAD_CONST %s" % (values, opname)
             self.add_unique_rule(rule, token.kind, token.attr, customize)
         elif opname == 'CALL_FUNCTION_EX_KW':
             self.addRule("""expr        ::= call_ex_kw

uncompyle6/scanner.py

@@ -242,17 +242,57 @@ class Scanner(object):
             pass
         return result_offset
 
-    def all_instr(self, start, end, instr, target=None, include_beyond_target=False):
+    def inst_matches(self, start, end, instr, target=None, include_beyond_target=False):
         """
-        Find all <instr> in the block from start to end.
-        <instr> is any python bytecode instruction or a list of opcodes
-        If <instr> is an opcode with a target (like a jump), a target
+        Find all `instr` in the block from start to end.
+        `instr` is a Python opcode or a list of opcodes
+        If `instr` is an opcode with a target (like a jump), a target
         destination can be specified which must match precisely.
 
         Return a list with indexes to them or [] if none found.
         """
+        try:
+            None in instr
+        except:
+            instr = [instr]
 
-        # FIXME: this is broken on 3.6+. Revise to use instructions self.insts
+        first = self.offset2inst_index[start]
+        result = []
+        for inst in self.insts[first:]:
+            if inst.opcode in instr:
+                if target is None:
+                    result.append(inst.offset)
+                else:
+                    t = self.get_target(inst.offset)
+                    if include_beyond_target and t >= target:
+                        result.append(inst.offset)
+                    elif t == target:
+                        result.append(inst.offset)
+                        pass
+                    pass
+                pass
+            if inst.offset >= end:
+                break
+            pass
+
+        # FIXME: put in a test
+        # check = self.all_instr(start, end, instr, target, include_beyond_target)
+        # assert result == check
+
+        return result
+
+
+    # FIXME: this is broken on 3.6+. Replace remaining (2.x-based) calls
+    # with inst_matches
+    def all_instr(self, start, end, instr, target=None, include_beyond_target=False):
+        """
+        Find all `instr` in the block from start to end.
+        `instr` is any Python opcode or a list of opcodes
+        If `instr` is an opcode with a target (like a jump), a target
+        destination can be specified which must match precisely.
+
+        Return a list with indexes to them or [] if none found.
+        """
 
         code = self.code
         assert(start >= 0 and end <= len(code))

uncompyle6/scanners/scanner2.py

@@ -152,7 +152,7 @@ class Scanner2(Scanner):
             #   raise AssertionError
             #  and
             #   assert ...
-            # Below we use the heuristic that it is preceded by a POP_JUMP.
+            # Below we use the heuristic that an "sssert" is preceded by a POP_JUMP.
             # however we could also use followed by RAISE_VARARGS
             # or for PyPy there may be a JUMP_IF_NOT_DEBUG before.
             # FIXME: remove uses of PJIF, and PJIT
@@ -420,7 +420,7 @@ class Scanner2(Scanner):
                                       (self.opc.PJIT, self.opc.JUMP_FORWARD),
                                       (self.opc.PJIT, self.opc.JUMP_ABSOLUTE)])
 
-        prelim = self.all_instr(start, end, self.stmt_opcodes)
+        prelim = self.all_instr(start, end, self.statement_opcodes)
 
         stmts = self.stmts = set(prelim)
         pass_stmts = set()

uncompyle6/scanners/scanner26.py

@@ -40,7 +40,7 @@ JUMP_OPS = opcode_26.JUMP_OPS
 class Scanner26(scan.Scanner2):
     def __init__(self, show_asm=False):
         super(Scanner26, self).__init__(2.6, show_asm)
-        self.stmt_opcodes = frozenset([
+        self.statement_opcodes = frozenset([
             self.opc.SETUP_LOOP,       self.opc.BREAK_LOOP,
             self.opc.SETUP_FINALLY,    self.opc.END_FINALLY,
             self.opc.SETUP_EXCEPT,     self.opc.POP_BLOCK,

uncompyle6/scanners/scanner27.py

@@ -23,7 +23,7 @@ class Scanner27(Scanner2):
         super(Scanner27, self).__init__(2.7, show_asm, is_pypy)
 
         # opcodes that start statements
-        self.stmt_opcodes = frozenset([
+        self.statement_opcodes = frozenset([
             self.opc.SETUP_LOOP,       self.opc.BREAK_LOOP,
             self.opc.SETUP_FINALLY,    self.opc.END_FINALLY,
             self.opc.SETUP_EXCEPT,

uncompyle6/scanners/scanner3.py

@@ -434,6 +434,7 @@ class Scanner3(Scanner):
                                   .opname == 'FOR_ITER'
                                   and self.insts[i+1].opname == 'JUMP_FORWARD')
 
+
                     if (is_continue or
                         (inst.offset in self.stmts and
                         (self.version != 3.0 or (hasattr(inst, 'linestart'))) and
@@ -616,7 +617,7 @@ class Scanner3(Scanner):
 
         # Compose preliminary list of indices with statements,
         # using plain statement opcodes
-        prelim = self.all_instr(start, end, self.statement_opcodes)
+        prelim = self.inst_matches(start, end, self.statement_opcodes)
 
         # Initialize final container with statements with
         # preliminary data
@@ -767,15 +768,17 @@ class Scanner3(Scanner):
                 if self.get_target(jump_back) >= next_line_byte:
                     jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE, start, False)
 
-                # This is wrong for 3.6+
-                if end > jump_back+4 and self.is_jump_forward(end):
-                    if self.is_jump_forward(jump_back+4):
-                        if self.get_target(jump_back+4) == self.get_target(end):
-                            self.fixed_jumps[offset] = jump_back+4
-                            end = jump_back+4
+                jb_inst = self.get_inst(jump_back)
+
+                jb_next_offset = self.next_offset(jb_inst.opcode, jump_back)
+                if end > jb_next_offset and self.is_jump_forward(end):
+                    if self.is_jump_forward(jb_next_offset):
+                        if self.get_target(jb_next_offset) == self.get_target(end):
+                            self.fixed_jumps[offset] = jb_next_offset
+                            end = jb_next_offset
                 elif target < offset:
-                    self.fixed_jumps[offset] = jump_back+4
-                    end = jump_back+4
+                    self.fixed_jumps[offset] = jb_next_offset
+                    end = jb_next_offset
 
                 target = self.get_target(jump_back)
 
@@ -877,11 +880,12 @@ class Scanner3(Scanner):
                             pass
                         else:
                             fix = None
-                            jump_ifs = self.all_instr(start, self.next_stmt[offset],
-                                                      self.opc.POP_JUMP_IF_FALSE)
+                            jump_ifs = self.inst_matches(start, self.next_stmt[offset],
+                                                         self.opc.POP_JUMP_IF_FALSE)
                             last_jump_good = True
                             for j in jump_ifs:
                                 if target == self.get_target(j):
+                                    # FIXME: remove magic number
                                     if self.lines[j].next == j + 3 and last_jump_good:
                                         fix = j
                                         break
@@ -914,7 +918,8 @@ class Scanner3(Scanner):
             if offset in self.ignore_if:
                 return
 
-            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+            rtarget_is_ja = code[pre_rtarget] == self.opc.JUMP_ABSOLUTE
+            if ( rtarget_is_ja and
                 pre_rtarget in self.stmts and
                 pre_rtarget != offset and
                 prev_op[pre_rtarget] != offset and
@@ -934,10 +939,13 @@ class Scanner3(Scanner):
             # or a conditional assignment like:
             #   x = 1 if x else 2
             #
+            # For 3.5, in addition the JUMP_FORWARD above we could have
+            # JUMP_BACK or CONTINUE
+            #
             # There are other contexts we may need to consider
             # like whether the target is "END_FINALLY"
             # or if the condition jump is to a forward location
-            if self.is_jump_forward(pre_rtarget):
+            if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5):
                 if_end = self.get_target(pre_rtarget)
 
                 # If the jump target is back, we are looping
@@ -1139,13 +1147,14 @@ class Scanner3(Scanner):
         assert(start>=0 and end<=len(self.code) and start <= end)
 
         # Find all offsets of requested instructions
-        instr_offsets = self.all_instr(start, end, instr, target, include_beyond_target)
+        instr_offsets = self.inst_matches(start, end, instr, target,
+                                          include_beyond_target)
         # Get all POP_JUMP_IF_TRUE (or) offsets
         if self.version == 3.0:
             jump_true_op = self.opc.JUMP_IF_TRUE
         else:
             jump_true_op = self.opc.POP_JUMP_IF_TRUE
-        pjit_offsets = self.all_instr(start, end, jump_true_op)
+        pjit_offsets = self.inst_matches(start, end, jump_true_op)
         filtered = []
         for pjit_offset in pjit_offsets:
             pjit_tgt = self.get_target(pjit_offset) - 3

uncompyle6/scanners/scanner30.py

@@ -193,11 +193,12 @@ class Scanner30(Scanner3):
                             pass
                         else:
                             fix = None
-                            jump_ifs = self.all_instr(start, self.next_stmt[offset],
-                                                      opc.JUMP_IF_FALSE)
+                            jump_ifs = self.inst_matches(start, self.next_stmt[offset],
+                                                         opc.JUMP_IF_FALSE)
                             last_jump_good = True
                             for j in jump_ifs:
                                 if target == self.get_target(j):
+                                    # FIXME: remove magic number
                                     if self.lines[j].next == j + 3 and last_jump_good:
                                         fix = j
                                         break

uncompyle6/semantics/consts.py

@@ -288,7 +288,10 @@ TABLE_DIRECT = {
     'except':           ( '%|except:\n%+%c%-', 3 ),
     'except_cond1':	    ( '%|except %c:\n', 1 ),
     'except_suite':     ( '%+%c%-%C', 0, (1, maxint, '') ),
+
+    # In Python 3.6, this is more complicated in the presence of "returns"
     'except_suite_finalize':     ( '%+%c%-%C', 1, (3, maxint, '') ),
+
     'pass':	            ( '%|pass\n', ),
     'STORE_FAST':	    ( '%{pattr}', ),
     'kv':		    ( '%c: %c', 3, 1 ),
@@ -381,7 +384,6 @@ PRECEDENCE = {
     'ret_cond_not':         28,
 
     '_mklambda':            30,
-    'call_kw':             100,  # 100 seems to to be module/function precidence
     'yield':               101,
     'yield_from':          101
 }

uncompyle6/semantics/customize.py

@@ -17,7 +17,7 @@
 """
 
 from uncompyle6.semantics.consts import (
-    INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT)
+    PRECEDENCE, INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT)
 
 from uncompyle6.semantics.make_function import (
     make_function3_annotate,
@@ -28,6 +28,7 @@ from xdis.code import iscode
 from uncompyle6.parsers.astnode import AST
 from uncompyle6.scanners.tok import Token
 from uncompyle6.semantics.helper import flatten_list
+from spark_parser.ast import GenericASTTraversalPruningException
 
 def customize_for_version(self, is_pypy, version):
     if is_pypy:
@@ -239,6 +240,45 @@ def customize_for_version(self, is_pypy, version):
             TABLE_DIRECT.update({
                 'LOAD_CLASSDEREF':	( '%{pattr}', ),
                 })
+
+            if version == 3.4:
+                def n_call(node):
+                    mapping = self._get_mapping(node)
+                    key = node
+                    for i in mapping[1:]:
+                        key = key[i]
+                        pass
+                    if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
+                        # We may want to fill this in...
+                        # But it is distinct from CALL_FUNCTION_VAR below
+                        pass
+                    elif key.kind.startswith('CALL_FUNCTION_VAR'):
+                        # CALL_FUNCTION_VAR's top element of the stack contains
+                        # the variable argument list, then comes
+                        # annotation args, then keyword args.
+                        # In the most least-top-most stack entry, but position 1
+                        # in node order, the positional args.
+                        argc = node[-1].attr
+                        nargs = argc & 0xFF
+                        kwargs = (argc >> 8) & 0xFF
+                        # FIXME: handle annotation args
+                        if kwargs != 0:
+                            # kwargs == 0 is handled by the table entry
+                            # Should probably handle it here though.
+                            if nargs == 0:
+                                template = ('%c(*%c, %C)',
+                                            0, -2, (1, kwargs+1, ', '))
+                            else:
+                                template = ('%c(%C, *%c, %C)',
+                                            0, (1, nargs+1, ', '),
+                                            -2, (-2-kwargs, -2, ', '))
+                            self.template_engine(template, node)
+                            self.prune()
+
+                    self.default(node)
+                self.n_call = n_call
+
+
             ########################
             # Python 3.5+ Additions
             #######################
@@ -281,11 +321,15 @@ def customize_for_version(self, is_pypy, version):
                             key = key[i]
                             pass
                         if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
-                            # Python 3.5 changes the stack position of *args. kwargs come
-                            # after *args whereas in earlier Pythons, *args is at the end
-                            # which simplifies things from our perspective.
-                            # Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
-                            # We will just swap the order to make it look like earlier Python 3.
+                            # Python 3.5 changes the stack position of
+                            # *args: kwargs come after *args whereas
+                            # in earlier Pythons, *args is at the end
+                            # which simplifies things from our
+                            # perspective.  Python 3.6+ replaces
+                            # CALL_FUNCTION_VAR_KW with
+                            # CALL_FUNCTION_EX We will just swap the
+                            # order to make it look like earlier
+                            # Python 3.
                             entry = table[key.kind]
                             kwarg_pos = entry[2][1]
                             args_pos = kwarg_pos - 1
@@ -296,7 +340,15 @@ def customize_for_version(self, is_pypy, version):
                                 args_pos = kwarg_pos
                                 kwarg_pos += 1
                         elif key.kind.startswith('CALL_FUNCTION_VAR'):
-                            nargs = node[-1].attr & 0xFF
+                            # CALL_FUNCTION_VAR's top element of the stack contains
+                            # the variable argument list, then comes
+                            # annotation args, then keyword args.
+                            # In the most least-top-most stack entry, but position 1
+                            # in node order, the positional args.
+                            argc = node[-1].attr
+                            nargs = argc & 0xFF
+                            kwargs = (argc >> 8) & 0xFF
+                            # FIXME: handle annotation args
                             if nargs > 0:
                                 template = ('%c(%C, ', 0, (1, nargs+1, ', '))
                             else:
@@ -304,15 +356,16 @@ def customize_for_version(self, is_pypy, version):
                             self.template_engine(template, node)
 
                             args_node =  node[-2]
-                            if args_node == 'pos_arg':
-                                args_node = args_node[0]
-                            if args_node == 'expr':
+                            if args_node in ('pos_arg', 'expr'):
                                 args_node = args_node[0]
                             if args_node == 'build_list_unpack':
                                 template = ('*%P)', (0, len(args_node)-1, ', *', 100))
                                 self.template_engine(template, args_node)
                             else:
-                                template = ('*%c)', -2)
+                                if len(node) - nargs > 3:
+                                    template = ('*%c, %C)', 1, (nargs+kwargs+1, -1, ', '))
+                                else:
+                                    template = ('*%c)', 1)
                                 self.template_engine(template, node)
                             self.prune()
 
@@ -353,6 +406,16 @@ def customize_for_version(self, is_pypy, version):
                 # Python 3.6+ Additions
                 #######################
 
+                # Value 100 is important; it is exactly
+                # module/function precidence.
+                PRECEDENCE['call_kw']     = 100
+                PRECEDENCE['call_kw36']   = 100
+                PRECEDENCE['call_ex']     = 100
+                PRECEDENCE['call_ex_kw']  = 100
+                PRECEDENCE['call_ex_kw2'] = 100
+                PRECEDENCE['call_ex_kw3'] = 100
+                PRECEDENCE['call_ex_kw4'] = 100
+
                 TABLE_DIRECT.update({
                     'tryfinally36':  ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
                                        (1, 'returns'), 3 ),
@@ -363,13 +426,14 @@ def customize_for_version(self, is_pypy, version):
                     'fstring_multi':  ( "f'''%c'''", 0),
                     'func_args36':    ( "%c(**", 0),
                     'try_except36':   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
+                    'except_return':  ( '%|except:\n%+%c%-', 3 ),
                     'unpack_list':    ( '*%c', (0, 'list') ),
                     'call_ex' : (
-                        '%c(%c)',
-                        (0, 'expr'), 1),
+                        '%c(%p)',
+                        (0, 'expr'), (1, 100)),
                     'call_ex_kw' : (
-                        '%c(%c)',
-                        (0, 'expr'), 2),
+                        '%c(%p)',
+                        (0, 'expr'), (2, 100)),
 
                 })
 
@@ -477,14 +541,17 @@ def customize_for_version(self, is_pypy, version):
                     kwargs = node[2]
                     if kwargs == 'expr':
                         kwargs = kwargs[0]
-                    self.write('**')
-                    self.preorder(kwargs)
+                    if kwargs == 'dict':
+                        self.call36_dict(kwargs)
+                    else:
+                        self.write('**')
+                        self.preorder(kwargs)
                     self.write(')')
                     self.prune()
                 self.n_call_ex_kw3 = call_ex_kw3
 
                 def call_ex_kw4(node):
-                    """Handle CALL_FUNCTION_EX 2  (have KW) but without
+                    """Handle CALL_FUNCTION_EX {1 or 2} but without
                     BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
                     self.preorder(node[0])
                     self.write('(')
@@ -503,8 +570,16 @@ def customize_for_version(self, is_pypy, version):
                     kwargs = node[2]
                     if kwargs == 'expr':
                         kwargs = kwargs[0]
-                    self.write('**')
-                    self.preorder(kwargs)
+                    call_function_ex = node[-1]
+                    assert call_function_ex == 'CALL_FUNCTION_EX_KW'
+                    # FIXME: decide if the below test be on kwargs == 'dict'
+                    if (call_function_ex.attr & 1 and
+                        (not isinstance(kwargs, Token) and kwargs != 'attribute')
+                        and not kwargs[0].kind.startswith('kvlist')):
+                        self.call36_dict(kwargs)
+                    else:
+                        self.write('**')
+                        self.preorder(kwargs)
                     self.write(')')
                     self.prune()
                 self.n_call_ex_kw4 = call_ex_kw4
@@ -559,8 +634,16 @@ def customize_for_version(self, is_pypy, version):
                         kv_node = node[0]
                         l = list(kv_node)
                         i = 0
+
+                        length = len(l)
+                        # FIXME: Parser-speed improved grammars will have BUILD_MAP
+                        # at the end. So in the future when everything is
+                        # complete, we can do an "assert" instead of "if".
+                        if kv_node[-1].kind.startswith("BUILD_MAP"):
+                            length -= 1
+
                         # Respect line breaks from source
-                        while i < len(l):
+                        while i < length:
                             self.write(sep)
                             name = self.traverse(l[i], indent='')
                             # Strip off beginning and trailing quotes in name
@@ -595,7 +678,11 @@ def customize_for_version(self, is_pypy, version):
                                 pass
                             pass
                     else:
-                        assert False, "Don't known to to untangle dictionary"
+                        self.write("**")
+                        try:
+                            self.default(node)
+                        except GenericASTTraversalPruningException:
+                            pass
 
                     self.prec = p
                     self.indent_less(INDENT_PER_LEVEL)
@@ -605,6 +692,22 @@ def customize_for_version(self, is_pypy, version):
 
                 FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
 
+                def n_except_suite_finalize(node):
+                    if node[1] == 'returns' and self.hide_internal:
+                        # Process node[1] only.
+                        # The code after "returns", e.g. node[3], is dead code.
+                        # Adding it is wrong as it dedents and another
+                        # exception handler "except_stmt" afterwards.
+                        # Note it is also possible that the grammar is wrong here.
+                        # and this should not be "except_stmt".
+                        self.indent_more()
+                        self.preorder(node[1])
+                        self.indent_less()
+                    else:
+                        self.default(node)
+                    self.prune()
+                self.n_except_suite_finalize = n_except_suite_finalize
+
                 def n_formatted_value(node):
                     if node[0] == 'LOAD_CONST':
                         self.write(node[0].attr)
@@ -639,40 +742,43 @@ def customize_for_version(self, is_pypy, version):
                 #     return
                 # self.n_kwargs_only_36 = kwargs_only_36
 
-                def kwargs_36(node):
-                    self.write('(')
-                    keys = node[-1].attr
+                def n_call_kw36(node):
+                    self.template_engine(("%c(", 0), node)
+                    keys = node[-2].attr
                     num_kwargs = len(keys)
-                    num_posargs = len(node) - (num_kwargs + 1)
+                    num_posargs = len(node) - (num_kwargs + 2)
                     n = len(node)
                     assert n >= len(keys)+1, \
                       'not enough parameters keyword-tuple values'
-                    # try:
-                    #     assert n >= len(keys)+1, \
-                    #         'not enough parameters keyword-tuple values'
-                    # except:
-                    #     from trepan.api import debug; debug()
                     sep = ''
-                    # FIXME: adjust output for line breaks?
-                    for i in range(num_posargs):
+
+                    line_number = self.line_number
+                    for i in range(1, num_posargs):
                         self.write(sep)
                         self.preorder(node[i])
-                        sep = ', '
+                        if line_number != self.line_number:
+                            sep = ",\n" + self.indent + "  "
+                        else:
+                            sep = ", "
+                        line_number = self.line_number
 
                     i = num_posargs
                     j = 0
                     # FIXME: adjust output for line breaks?
-                    while i < n-1:
+                    while i < n-2:
                         self.write(sep)
                         self.write(keys[j] + '=')
                         self.preorder(node[i])
-                        sep=', '
+                        if line_number != self.line_number:
+                            sep = ",\n" + self.indent + "  "
+                        else:
+                            sep = ", "
                         i += 1
                         j += 1
                     self.write(')')
                     self.prune()
                     return
-                self.n_kwargs_36 = kwargs_36
+                self.n_call_kw36 = n_call_kw36
 
                 def starred(node):
                     l = len(node)
@@ -681,9 +787,18 @@ def customize_for_version(self, is_pypy, version):
                     if pos_args == 'expr':
                         pos_args = pos_args[0]
                     if pos_args == 'tuple':
+                        build_tuple = pos_args[0]
+                        if build_tuple.kind.startswith('BUILD_TUPLE'):
+                            tuple_len = 0
+                        else:
+                            tuple_len = len(node) - 1
                         star_start = 1
                         template = '%C', (0, -1, ', ')
                         self.template_engine(template, pos_args)
+                        if tuple_len == 0:
+                            self.write("*()")
+                            # That's it
+                            self.prune()
                         self.write(', ')
                     else:
                         star_start = 0
@@ -691,6 +806,7 @@ def customize_for_version(self, is_pypy, version):
                         template = ( '*%C', (star_start, -1, ', *') )
                     else:
                         template = ( '*%c', (star_start, 'expr') )
+
                     self.template_engine(template, node)
                     self.prune()
 

uncompyle6/semantics/make_function.py

@@ -428,10 +428,7 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None):
                                                     code, self.version)
 
     # Python 2 doesn't support the "nonlocal" statement
-    try:
-        assert self.version >= 3.0 or not nonlocals
-    except:
-        from trepan.api import debug; debug()
+    assert self.version >= 3.0 or not nonlocals
 
     for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
@@ -512,15 +509,22 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
     if isinstance(args_node.attr, tuple):
         pos_args, kw_args, annotate_argc  = args_node.attr
         # FIXME: there is probably a better way to classify this.
+        have_kwargs = node[0].kind.startswith('kwarg') or node[0] == 'no_kwargs'
+        if len(node) >= 4:
+            lc_index = -4
+        else:
+            lc_index = -3
+            pass
+
         if (self.version <= 3.3 and len(node) > 2 and
-            node[lambda_index] != 'LOAD_LAMBDA' and
-            (node[0].kind.startswith('kwarg') or node[-4].kind != 'load_closure')):
+                node[lambda_index] != 'LOAD_LAMBDA' and
+                (have_kwargs or node[lc_index].kind != 'load_closure')):
             # args are after kwargs; kwargs are bundled as one node
             defparams = node[1:args_node.attr[0]+1]
         else:
             # args are before kwargs; kwags as bundled as one node
             defparams = node[:args_node.attr[0]]
-        pos_args, kw_args, annotate_argc  = args_node.attr
+            pass
     else:
         if self.version < 3.6:
             defparams = node[:args_node.attr]
@@ -528,8 +532,10 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         else:
             default, kw_args, annotate, closure = args_node.attr
             if default:
-                assert node[0] == 'expr', "expecting mkfunc default node to be an expr"
                 expr_node = node[0]
+                if node[0] == 'pos_arg':
+                    expr_node = expr_node[0]
+                assert expr_node == 'expr', "expecting mkfunc default node to be an expr"
                 if (expr_node[0] == 'LOAD_CONST' and
                     isinstance(expr_node[0].attr, tuple)):
                     defparams = [repr(a) for a in expr_node[0].attr]
@@ -537,7 +543,21 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
                     defparams =  [self.traverse(n, indent='') for n in expr_node[0][:-1]]
             else:
                 defparams = []
-            # FIXME: handle kw, annotate and closure
+
+            i = -4
+            kw_pairs = 0
+            if closure:
+                # FIXME: fill in
+                i -= 1
+            if annotate:
+                # FIXME: fill in
+                i -= 1
+            if kw_args:
+                kw_node = node[i]
+                if kw_node == 'expr':
+                    kw_node = kw_node[0]
+                if kw_node == 'dict':
+                    kw_pairs = kw_node[-1].attr
         pass
 
     if 3.0 <= self.version <= 3.2:
@@ -576,10 +596,10 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         return
 
     if self.version >= 3.0:
-        kw_pairs = args_node.attr[1]
+        if self.version < 3.6:
+            kw_pairs = args_node.attr[1]
     else:
         kw_pairs = 0
-    indent = self.indent
 
     # build parameters
     params = []
@@ -695,7 +715,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
             for n in node:
                 if n == 'pos_arg':
                     continue
-                elif self.version >= 3.4 and not (n.kind in ('kwargs', 'kwargs1', 'kwarg')):
+                elif self.version >= 3.4 and not (n.kind in ('kwargs', 'no_kwargs', 'kwarg')):
                     continue
                 else:
                     self.preorder(n)

uncompyle6/semantics/pysource.py

@@ -404,7 +404,7 @@ class SourceWalker(GenericASTTraversal, object):
 
             def n_mkfunc_annotate(node):
 
-                if self.version >= 3.3 or node[-2] == 'kwargs':
+                if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
                     # LOAD_CONST code object ..
                     # LOAD_CONST        'x0'  if >= 3.3
                     # EXTENDED_ARG
@@ -468,73 +468,6 @@ class SourceWalker(GenericASTTraversal, object):
 
                     })
 
-                    def async_call(node):
-                        self.f.write('async ')
-                        node.kind == 'call'
-                        p = self.prec
-                        self.prec = 80
-                        self.template_engine(('%c(%P)', 0,
-                                              (1, -4, ', ', 100)), node)
-                        self.prec = p
-                        node.kind == 'async_call'
-                        self.prune()
-                    self.n_async_call = async_call
-                    self.n_build_list_unpack = self.n_list
-
-                    if version == 3.5:
-                        def n_call(node):
-                            mapping = self._get_mapping(node)
-                            table = mapping[0]
-                            key = node
-                            for i in mapping[1:]:
-                                key = key[i]
-                                pass
-                            if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
-                                # Python 3.5 changes the stack position of *args. kwargs come
-                                # after *args whereas in earlier Pythons, *args is at the end
-                                # which simpilfiies things from our perspective.
-                                # Python 3.6+ replaces CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
-                                # We will just swap the order to make it look like earlier Python 3.
-                                entry = table[key.kind]
-                                kwarg_pos = entry[2][1]
-                                args_pos = kwarg_pos - 1
-                                # Put last node[args_pos] after subsequent kwargs
-                                while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
-                                    # swap node[args_pos] with node[kwargs_pos]
-                                    node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
-                                    args_pos = kwarg_pos
-                                    kwarg_pos += 1
-                            self.default(node)
-                        self.n_call = n_call
-
-                    def n_function_def(node):
-                        if self.version == 3.6:
-                            code_node = node[0][0]
-                        else:
-                            code_node = node[0][1]
-
-                        is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
-                        if (is_code and
-                            (code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
-                            self.template_engine(('\n\n%|async def %c\n',
-                                                  -2), node)
-                        else:
-                            self.template_engine(('\n\n%|def %c\n', -2),
-                                                 node)
-                        self.prune()
-                    self.n_function_def = n_function_def
-
-                    def unmapexpr(node):
-                        last_n = node[0][-1]
-                        for n in node[0]:
-                            self.preorder(n)
-                            if n != last_n:
-                                self.f.write(', **')
-                                pass
-                            pass
-                        self.prune()
-                        pass
-                    self.n_unmapexpr = unmapexpr
 
                 pass # version >= 3.4
             pass # version >= 3.0
@@ -1041,7 +974,7 @@ class SourceWalker(GenericASTTraversal, object):
 
     def n_mkfunc(self, node):
 
-        if self.version >= 3.3 or node[-2] == 'kwargs':
+        if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
             # LOAD_CONST code object ..
             # LOAD_CONST        'x0'  if >= 3.3
             # MAKE_FUNCTION ..
@@ -1057,7 +990,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.write(func_name)
 
         self.indent_more()
-        self.make_function(node, is_lambda=False, codeNode=code_node)
+        self.make_function(node, is_lambda=False, code_node=code_node)
 
         if len(self.param_stack) > 1:
             self.write('\n\n')
@@ -1067,14 +1000,14 @@ class SourceWalker(GenericASTTraversal, object):
         self.prune() # stop recursing
 
     def make_function(self, node, is_lambda, nested=1,
-                      codeNode=None, annotate=None):
+                      code_node=None, annotate=None):
         if self.version >= 3.0:
-            make_function3(self, node, is_lambda, nested, codeNode)
+            make_function3(self, node, is_lambda, nested, code_node)
         else:
-            make_function2(self, node, is_lambda, nested, codeNode)
+            make_function2(self, node, is_lambda, nested, code_node)
 
     def n_mklambda(self, node):
-        self.make_function(node, is_lambda=True, codeNode=node[-2])
+        self.make_function(node, is_lambda=True, code_node=node[-2])
         self.prune() # stop recursing
 
     def n_list_comp(self, node):
@@ -1563,7 +1496,7 @@ class SourceWalker(GenericASTTraversal, object):
 
             assert 'mkfunc' == build_class[1]
             mkfunc = build_class[1]
-            if mkfunc[0] == 'kwargs':
+            if mkfunc[0] in ('kwargs', 'no_kwargs'):
                 if 3.0 <= self.version <= 3.2:
                     for n in mkfunc:
                         if hasattr(n, 'attr') and iscode(n.attr):
@@ -1608,8 +1541,15 @@ class SourceWalker(GenericASTTraversal, object):
                 subclass_info = node
                 subclass_code = build_class[1][0].attr
             elif not subclass_info:
-                subclass_code = build_class[1][0].attr
-                subclass_info = node[0]
+                if mkfunc[0] in ('no_kwargs', 'kwargs'):
+                    subclass_code = mkfunc[1].attr
+                else:
+                    subclass_code = mkfunc[0].attr
+                if node == 'classdefdeco2':
+                    subclass_info = node
+                else:
+                    subclass_info = node[0]
+
         else:
             if node == 'classdefdeco2':
                 build_class = node
@@ -1680,7 +1620,7 @@ class SourceWalker(GenericASTTraversal, object):
             self.write(')')
 
     def print_super_classes3(self, node):
-        n = len(node)-1
+        n = len(node) - 1
         if node.kind != 'expr':
             assert node[n].kind.startswith('CALL_FUNCTION')
 
@@ -1756,7 +1696,8 @@ class SourceWalker(GenericASTTraversal, object):
 
         self.indent_more(INDENT_PER_LEVEL)
         sep = INDENT_PER_LEVEL[:-1]
-        self.write('{')
+        if node[0] != 'dict_entry':
+            self.write('{')
         line_number = self.line_number
 
         if self.version >= 3.0 and not self.is_pypy:
@@ -1764,9 +1705,13 @@ class SourceWalker(GenericASTTraversal, object):
                 # Python 3.5+ style key/value list in dict
                 kv_node = node[0]
                 l = list(kv_node)
+                length = len(l)
+                if kv_node[-1].kind.startswith("BUILD_MAP"):
+                    length -= 1
                 i = 0
+
                 # Respect line breaks from source
-                while i < len(l):
+                while i < length:
                     self.write(sep)
                     name = self.traverse(l[i], indent='')
                     if i > 0:
@@ -1776,7 +1721,7 @@ class SourceWalker(GenericASTTraversal, object):
                     self.write(name, ': ')
                     value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
                     self.write(value)
-                    sep = ","
+                    sep = ", "
                     if line_number != self.line_number:
                         sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
                         line_number = self.line_number
@@ -1803,7 +1748,7 @@ class SourceWalker(GenericASTTraversal, object):
                     self.write(name, ': ')
                     value = self.traverse(l[i], indent=self.indent+(len(name)+2)*' ')
                     self.write(value)
-                    sep = ","
+                    sep = ", "
                     if line_number != self.line_number:
                         sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
                         line_number = self.line_number
@@ -1823,7 +1768,7 @@ class SourceWalker(GenericASTTraversal, object):
                     line_number = self.line_number
                     self.write(':')
                     self.write(self.traverse(value[0]))
-                    sep = ","
+                    sep = ", "
                     if line_number != self.line_number:
                         sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
                         line_number = self.line_number
@@ -1834,15 +1779,38 @@ class SourceWalker(GenericASTTraversal, object):
                 if sep.startswith(",\n"):
                     self.write(sep[1:])
                 pass
+            elif node[0].kind.startswith('dict_entry'):
+                assert self.version >= 3.5
+                template = ("%C", (0, len(node[0]), ", **"))
+                self.template_engine(template, node[0])
+                sep = ''
+            elif (node[-1].kind.startswith('BUILD_MAP_UNPACK')
+                  or node[-1].kind.startswith('dict_entry')):
+                assert self.version >= 3.5
+                # FIXME: I think we can intermingle dict_comp's with other
+                # dictionary kinds of things. The most common though is
+                # a sequence of dict_comp's
+                kwargs = node[-1].attr
+                template = ("**%C", (0, kwargs, ", **"))
+                self.template_engine(template, node)
+                sep = ''
+
             pass
         else:
-            # Python 2 style kvlist
-            assert node[-1].kind.startswith('kvlist')
-            kv_node = node[-1] # goto kvlist
+            # Python 2 style kvlist. Find beginning of kvlist.
+            if node[0].kind.startswith("BUILD_MAP"):
+                if len(node) > 1 and node[1].kind in ('kvlist', 'kvlist_n'):
+                    kv_node = node[1]
+                else:
+                    kv_node = node[1:]
+            else:
+                assert node[-1].kind.startswith('kvlist')
+                kv_node = node[-1]
 
             first_time = True
             for kv in kv_node:
                 assert kv in ('kv', 'kv2', 'kv3')
+
                 # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
                 # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
                 # kv3 ::= expr expr STORE_MAP
@@ -1882,7 +1850,7 @@ class SourceWalker(GenericASTTraversal, object):
                     value = self.traverse(kv[0], indent=self.indent+(len(name)+2)*' ')
                     pass
                 self.write(value)
-                sep = ","
+                sep = ", "
                 if line_number != self.line_number:
                     sep += "\n" + self.indent + "  "
                     line_number = self.line_number
@@ -1891,7 +1859,8 @@ class SourceWalker(GenericASTTraversal, object):
             pass
         if sep.startswith(",\n"):
             self.write(sep[1:])
-        self.write('}')
+        if node[0] != 'dict_entry':
+            self.write('}')
         self.indent_less(INDENT_PER_LEVEL)
         self.prec = p
         self.prune()
@@ -2182,6 +2151,10 @@ class SourceWalker(GenericASTTraversal, object):
                 TABLE_R[k] = ('%c(%P)', 0, (1, -1, ', ', 100))
             elif op in ('CALL_FUNCTION_VAR',
                         'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
+
+                # FIXME: handle everything in customize.
+                # Right now, some of this is here, and some in that.
+
                 if v == 0:
                     str = '%c(%C' # '%C' is a dummy here ...
                     p2 = (0, 0, None) # .. because of the None in this
@@ -2196,6 +2169,15 @@ class SourceWalker(GenericASTTraversal, object):
                             entry = ('%c(*%C, %c)', 0, p2, -2)
                         elif str == '%c(%C':
                             entry = ('%c(*%C)', 0, (1, 100, ''))
+                    elif self.version == 3.4:
+                        # CALL_FUNCTION_VAR's top element of the stack contains
+                        # the variable argument list
+                        if v == 0:
+                            str = '%c(*%c)'
+                            entry = (str, 0, -2)
+                        else:
+                            str = '%c(%C, *%c)'
+                            entry = (str, 0, p2, -2)
                     else:
                         str += '*%c)'
                         entry = (str, 0, p2, -2)

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.1.0'
+VERSION='3.1.1'