Merge branch 'master' into python-2.4

which are created via a call to create a subclass.

Should be more general though.

.gitignore

@@ -21,3 +21,4 @@
 ChangeLog
 __pycache__
 build
+nohup.out

Makefile

@@ -27,19 +27,20 @@ check:
 check-short: pytest
 	$(MAKE) -C test check-short
 
+# Note for 2.6 use <=3.0.1 see requirements-dev.txt
 #: Tests for Python 2.7, 3.3 and 3.4
-check-2.7 check-3.3 check-3.4: pytest
+check-2.6 check-2.7 check-3.3 check-3.4 check-3.5: pytest
 	$(MAKE) -C test $@
 
 #: Tests for Python 3.2 and 3.5 - pytest doesn't work here
 # Or rather 3.5 doesn't work not on Travis
-check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
+check-3.0 check-3.1 check-3.2 check-3.6:
 	$(MAKE) -C test $@
 
 check-3.7: pytest
 
-#:Tests for Python 2.6 (doesn't have pytest)
-check-2.4 check-2.5 check-2.6:
+#:Tests for Python 2.4-2.5 (don't have pytest)
+check-2.4 check-2.5:
 	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0

NEWS

@@ -1,3 +1,32 @@
+uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
+
+- Python 3.x subclass and call parsing fixes
+- Allow/note running on Python 3.1
+- improve 3.5+ BUILD_MAP_UNPACK
+- DRY instruction building code between 2.x and 3.x
+- expand testing
+
+uncompyle6 3.1.1 2018-04-01 Easter April Fool's
+
+Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
+
+- fill out 3.5+ BUILD_MAP_UNPACK (more work is needed)
+- fill out 3.4+ CALL_FUNCTION_... (more work is needed)
+- fill out 3.5 MAKE_FUNCTION  (more work is needed)
+- reduce 3.5, 3.6 control-flow bugs
+- reduce ambiguity in rules that lead to long (exponential?) parses
+- limit/isolate some 2.6/2.7,3.x grammar rules
+- more runtime testing of decompiled code
+- more removal of parenthesis around calls via setting precidence
+
+uncompyle6 3.1.0 2018-03-21 Equinox
+
+- Add code_deparse_with_offset() fragment function.
+- Correct paramenter call fragment deparse_code()
+- Lots of 3.6, 3.x, and 2.7 bug fixes
+  About 5% of 3.6 fail parsing now. But
+  semantics still needs much to be desired.
+
 uncompyle6 3.0.1 2018-02-17
 
 - All Python 2.6.9 standard library files weakly verify

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.4.8'
+export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'

pytest/test_fjt.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
-from xdis.bytecode import Bytecode
-from array import array
 def bug(state, slotstate):
     if state:
         if slotstate is not None:
@@ -25,14 +23,7 @@ def test_if_in_for():
     code = bug.func_code
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
-        scan.build_prev_op(n)
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
@@ -47,14 +38,7 @@ def test_if_in_for():
         #    {'start': 62, 'end': 63, 'type': 'for-else'}]
 
         code = bug_loop.__code__
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.build_prev_op(n)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
         assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
         assert scan.structs == [
@@ -68,14 +52,7 @@ def test_if_in_for():
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
     elif 3.2 < PYTHON_VERSION <= 3.4:
-        bytecode = Bytecode(code, scan.opc)
-        scan.code = array('B', code.co_code)
-        scan.lines = scan.build_lines_data(code)
-        scan.build_prev_op()
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
@@ -85,5 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        assert True, "FIXME: should note fixed"
+        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        assert True
     return

pytest/test_grammar.py

@@ -18,40 +18,44 @@ def test_grammar():
      right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['expr1024', 'pos_arg', 'get_iter', 'attribute'])
+    expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
     unused_rhs = set(['list', 'mkfunc',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
-    if PYTHON3:
-        expect_lhs.add('load_genexpr')
+
+    if PYTHON_VERSION > 2.6:
         expect_lhs.add('kvlist')
         expect_lhs.add('kv3')
+        unused_rhs.add('dict')
+
+    if PYTHON3:
+        expect_lhs.add('load_genexpr')
 
         unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp classdefdeco2
-        dict
+        except_pop_except generator_exp
         """.split()))
         if PYTHON_VERSION >= 3.0:
             expect_lhs.add("annotate_arg")
             expect_lhs.add("annotate_tuple")
             unused_rhs.add("mkfunc_annotate")
             unused_rhs.add('call')
+            unused_rhs.add("dict_comp")
+            unused_rhs.add("classdefdeco1")
             if PYTHON_VERSION < 3.6:
                 # 3.6 has at least one non-custom call rule
                 # the others don't
                 unused_rhs.add('call')
                 if PYTHON_VERSION == 3.5:
                     expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
                     pass
                 pass
             else:
                 expect_right_recursive.add((('l_stmts',
                                              ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                # expect_lhs.add('kwargs1')
                 pass
             pass
         pass
@@ -85,6 +89,8 @@ def test_grammar():
             """.split())
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
+        if PYTHON_VERSION == 2.6:
+            opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
         ignore_set.add('LOAD_CLASSNAME')

pytest/test_single_compile.py

@@ -1,6 +1,6 @@
 from uncompyle6 import PYTHON_VERSION, deparse_code
 
-if PYTHON_VERSION >= 2.5:
+if PYTHON_VERSION >= 2.6:
     def test_single_mode():
         single_expressions = (
             'i = 1',

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest>=3.0.0
+pytest>=3.0.0,<=3.0.1
 flake8
-hypothesis<=3.8.3
+hypothesis<=3.0.0

test/.gitignore

@@ -0,0 +1 @@
+/nohup.out

test/Makefile

@@ -3,9 +3,9 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-5.6 5.6 5.8 \
-      grammar-coverage-2.5 grammar-coverage-2.6 grammarcoverage-2.7 \
-      grammar-coverage-3.1 grammar-coverage-3.2 grammarcoverage-3.3 \
-      grammar-coverage-3.4 grammar-coverage-3.5 grammarcoverage-3.6
+      grammar-coverage-2.5 grammar-coverage-2.6 grammar-coverage-2.7 \
+      grammar-coverage-3.1 grammar-coverage-3.2 grammar-coverage-3.3 \
+      grammar-coverage-3.4 grammar-coverage-3.5 grammar-coverage-3.6
 
 
 GIT2CL ?= git2cl
@@ -37,18 +37,22 @@ check-3.0: check-bytecode
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
 	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
@@ -117,26 +121,26 @@ check-bytecode-2.5:
 #: Get grammar coverage for Python 2.4
 grammar-coverage-2.4:
 	-rm $(COVER_DIR)/spark-grammar-24.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pyenvlib.py --2.4.6 --max= 800
 
 #: Get grammar coverage for Python 2.5
 grammar-coverage-2.5:
-	-rm $(COVER_DIR)/spark-grammar-25.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+	-rm $(COVER_DIR)/spark-grammar-2.5.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pyenvlib.py --2.5.6 --max=800
 
 #: Get grammar coverage for Python 2.6
 grammar-coverage-2.6:
-	-rm $(COVER_DIR)/spark-grammar-26.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+	-rm $(COVER_DIR)/spark-grammar-2.6.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pyenvlib.py --2.6.9 --max=800
 
 #: Get grammar coverage for Python 2.7
 grammar-coverage-2.7:
-	-rm $(COVER_DIR)/spark-grammar-27.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -147,33 +151,39 @@ SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-30.cover $(PYTHON) test_pythonl
 
 #: Get grammar coverage for Python 3.1
 grammar-coverage-3.1:
-	-rm $(COVER_DIR)/spark-grammar-31.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pyenvlib.py --3.1.5
+	-rm $(COVER_DIR)/spark-grammar-3.1.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pyenvlib.py --3.1.5
 
 #: Get grammar coverage for Python 3.2
 grammar-coverage-3.2:
-	-rm $(COVER_DIR)/spark-grammar-32.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
+	-rm $(COVER_DIR)/spark-grammar-3.2.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pyenvlib.py --3.2.6
 
 #: Get grammar coverage for Python 3.3
 grammar-coverage-3.3:
-	-rm $(COVER_DIR)/spark-grammar-33.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pyenvlib.py --3.3.6
+	-rm $(COVER_DIR)/spark-grammar-3.3.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pyenvlib.py --3.3.7 --max=800
 
 #: Get grammar coverage for Python 3.4
 grammar-coverage-3.4:
-	-rm $(COVER_DIR)/spark-grammar-34.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2
+	-rm $(COVER_DIR)/spark-grammar-3.4.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pyenvlib.py --3.4.8 --max=800
 
 #: Get grammar coverage for Python 3.5
 grammar-coverage-3.5:
-	rm $(COVER_DIR)/spark-grammar-35.cover || /bin/true
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pyenvlib.py --3.5.3
+	rm $(COVER_DIR)/spark-grammar-3.5.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pyenvlib.py --3.5.5 --max=450
+
+#: Get grammar coverage for Python 3.6
+grammar-coverage-3.6:
+	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
@@ -192,10 +202,12 @@ check-bytecode-3.0:
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:

test/grammar-cover/.gitignore

@@ -0,0 +1 @@
+/.python-version

test/grammar-cover/README.md

@@ -0,0 +1 @@
+Code in this directory gets statistics on grammar coverage

test/grammar-cover/convert.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+for VERS in 2{4,5,6,7} 3{2,3,4,5} ; do
+    GRAMMAR_TXT=grammar-${VERS}.txt
+    spark-parser-coverage --max-count 3000 --path spark-grammar-${VERS}.cover > $GRAMMAR_TXT
+done

test/grammar-cover/grammar-all.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+$SHELL ./grammar.sh 2.4 2.5 2.6 2.7 3.2 3.3 3.4 3.5 3.6

test/grammar-cover/grammar.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Remake Python grammar statistics
+
+typeset -A ALL_VERS=([2.4]=2.4.6 [2.5]=2.5.6 [2.6]=2.6.9 [2.7]=2.7.14 [3.2]=3.2.6 [3.3]=3.3.6 [3.4]=3.4.8 [3.5]=3.5.5 [3.6]=3.6.4)
+
+if (( $# == 0 )); then
+    echo 1>&2 "usage: $0 two-digit-version"
+    exit 1
+fi
+
+me=${BASH_SOURCE[0]}
+workdir=$(dirname $me)
+cd $workdir
+workdir=$(pwd)
+while [[ -n $1 ]]  ; do
+    SHORT_VERSION=$1; shift
+    LONG_VERSION=${ALL_VERS[$SHORT_VERSION]}
+    if [[ -z ${LONG_VERSION} ]] ; then
+       echo 1>&2 "Version $SHORT_VERSION not known"
+       exit 2
+    fi
+
+    tmpdir=$workdir/../../tmp/grammar-cover
+    COVER_FILE=${tmpdir}/spark-grammar-${SHORT_VERSION}.cover
+    [[ -d  $tmpdir ]] || mkdir $tmpdir
+    cd $workdir/../..
+    if [[ $SHORT_VERSION > 2.5 ]] ; then
+        source ./admin-tools/setup-master.sh
+    else
+        source ./admin-tools/setup-python-2.4.sh
+    fi
+    GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
+    (cd ../.. && pyenv local ${LONG_VERSION})
+    cd ./test
+    if [[ -r $COVER_FILE ]]; then
+	rm $COVER_FILE
+    fi
+    if [[ -r $GRAMMAR_TXT ]]; then
+        GRAMMAR_SAVE_TXT=${tmpdir}/grammar-${SHORT_VERSION}-save.txt
+        cp $GRAMMAR_TXT $GRAMMAR_SAVE_TXT
+    fi
+    make grammar-coverage-${SHORT_VERSION};
+    spark-parser-coverage --max-count=3000 --path $COVER_FILE > $GRAMMAR_TXT
+done

test/grammar-cover/run-and-email.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+USER=${USER:-rocky}
+EMAIL=${EMAIL:-rb@dustyfeet.com}
+SUBJECT_PREFIX="grammar cover testing for"
+LOGFILE=/tmp/grammar-cover-$$.log
+/bin/bash ./grammar-all.sh  >$LOGFILE 2>&1
+rc=$?
+if ((rc == 0)); then
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX ok" ${USER}@localhost
+else
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX not ok" ${USER}@localhost
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX not ok" $EMAIL
+fi

test/run-and-email.sh

@@ -1,13 +1,69 @@
-#!/usr/bin/bash
+#!/bin/bash
+
+function displaytime {
+    printf "ran in "
+    local T=$1
+    local D=$((T/60/60/24))
+    local H=$((T/60/60%24))
+    local M=$((T/60%60))
+    local S=$((T%60))
+    (( $D > 0 )) && printf '%d days ' $D
+    (( $H > 0 )) && printf '%d hours ' $H
+    (( $M > 0 )) && printf '%d minutes ' $M
+    (( $D > 0 || $H > 0 || $M > 0 )) && printf 'and '
+    printf '%d seconds\n' $S
+}
+
+PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.3.7 3.4.8 2.6.9 3.6.4"}
+# PYVERSION=${PYVERSION:-"3.5.5"}
+
+USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-for VERSION in 2.7.14 2.6.9 ; do
-    LOGFILE=/tmp/pyenlib-$VERSION-$$.log
-    python ./test_pyenvlib.py --max 800 --weak-verify --$VERSION  >$LOGFILE 2>&1
-    rc=$?
-    if ((rc == 0)); then
-	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+MAX_TESTS=${MAX_TESTS:-800}
+typeset -i RUN_STARTTIME=$(date +%s)
+
+for VERSION in $PYVERSION ; do
+    typeset -i rc=0
+    LOGFILE=/tmp/pyenvlib-$VERSION-$$.log
+
+    if [[ $VERSION == '3.5.5' ]] ; then
+	MAX_TESTS=224
+    elif [[ $VERSION == '3.2.6' ]] ; then
+	MAX_TESTS=700
+    elif [[ $VERSION == '3.6.4' ]] ; then
+	MAX_TESTS=400
     else
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+	MAX_TESTS=800
     fi
+
+    if ! pyenv local $VERSION ; then
+	rc=1
+    else
+      echo Python Version $(pyenv local) > $LOGFILE
+      echo "" >> $LOGFILE
+      typeset -i ALL_FILES_STARTTIME=$(date +%s)
+      python ./test_pyenvlib.py --max ${MAX_TESTS} --weak-verify --$VERSION  >>$LOGFILE 2>&1
+      rc=$?
+
+      echo Python Version $(pyenv local) >> $LOGFILE
+      echo "" >>$LOGFILE
+
+      typeset -i ALL_FILES_ENDTIME=$(date +%s)
+      (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
+      displaytime $time_diff >> $LOGFILE
+    fi
+
+    SUBJECT_PREFIX="pyenv weak verify (max $MAX_TESTS) for"
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
+    else
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${EMAIL}
+    fi
+    rm .python-version
 done
+
+typeset -i RUN_ENDTIME=$(date +%s)
+(( time_diff =  RUN_ENDTIME - RUN_STARTTIME))
+elapsed_time=$(displaytime $time_diff)
+echo "Run complete $elapsed_time for versions $PYVERSION" | mail -s "pyenv weak verify in $elapsed_time" ${EMAIL}

test/simple_source/bug25/04_try_except_else.py

@@ -0,0 +1,18 @@
+# Bug found in 2.4 test_math.py
+# Bug was turning last try/except/else into try/else
+import math
+def test_exceptions():
+    try:
+        x = math.exp(-1000000000)
+    except:
+        raise RuntimeError
+
+    x = 1
+    try:
+        x = math.sqrt(-1.0)
+    except ValueError:
+        return x
+    else:
+        raise RuntimeError
+
+test_exceptions()

test/simple_source/bug27+/02_assert.py

@@ -0,0 +1,31 @@
+# From 2.7 test_argparse.py
+# Bug was turning assert into an "or raise" statement
+def __call__(arg, dest):
+    try:
+        assert arg == 'spam', 'dest: %s' % dest
+    except:
+        raise
+
+__call__('spam', __file__)
+
+# From python 2.7.14 lib2to3/refactor.py
+# Bug was mangling assert turning if into "or"
+def refactor_doctest(clipped, new):
+    assert clipped, clipped
+    if not new:
+        new += u"\n"
+    return
+
+# From 2.7.14 test_hashlib.py
+# The bug was turning assert into an "if"
+# statement which isn't wrong, but we got the
+# range of the if incorrect. When we have
+# better control flow analysis we can revisit.
+def test_threaded_hashing():
+    for threadnum in xrange(1):
+        result = 1
+        assert result > 0
+        result = 2
+    return result
+
+assert test_threaded_hashing() == 2

test/simple_source/bug27+/05_not_unconditional.py

@@ -1,5 +1,5 @@
 # From 2.7 test_itertools.py
-# Bug was in 2.7 decompiling like the commented out
+# Bug was in 2.7 decompiling the target assignment
 # code below
 from itertools import izip_longest
 for args in [

test/simple_source/bug31/05_abc_test.py

@@ -0,0 +1,18 @@
+# Extracted from Python 3.5 test_abc.py
+# Bug is class having only a single kwarg
+# subclass.
+import abc
+import unittest
+from inspect import isabstract
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract
+            def bar(self): pass  # concrete
+        assert C.__abstractmethods__, {"foo"}
+        assert isabstract(C)
+        pass
+
+test_abstractmethod_integration(None)

test/simple_source/bug33/02_named_and_kwargs.py

@@ -1,5 +1,6 @@
 # Python 3.6 subprocess.py bug
 # Bug is getting params correct: timeout before **kwargs
+import subprocess
 def call(*popenargs, timeout=None, **kwargs):
     return
 
@@ -14,6 +15,9 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
 # From 3.4 asyncio/locks.py
 # Bug was handling" "value=1, *"
 
+class Semaphore:
+    pass
+
 class BoundedSemaphore(Semaphore):
     def __init__(self, value=1, *, loop=None):
         super().__init__(value, loop=loop)

test/simple_source/bug33/05_nonlocal.py

@@ -0,0 +1,12 @@
+# From Python 3.6 functools.py
+# Bug was in detecting "nonlocal" access
+def not_bug():
+    cache_token = 5
+
+    def register():
+        nonlocal cache_token
+        return cache_token == 5
+
+    return register()
+
+assert not_bug()

test/simple_source/bug34/05_while1_if_continue.py

@@ -0,0 +1,37 @@
+# Bug in Python 3.4 text_file.py
+# Bug is handling:  while true ... if ... continue
+def readline(b):
+    a = 1
+    while True:
+        if b:
+            if b[0]:
+                a = 2
+                b = None
+                continue
+        b = None
+        a = 5
+
+        return a
+
+assert readline(None) == 1
+assert readline([2]) == 2
+
+def readline2(self):
+    while True:
+        line = 5
+        if self[0]:
+            if self:
+                self[0] = 1
+                continue
+
+        return line + self[0]
+
+# From 3.4.4 connection.py
+def PipeClient(address):
+    while 1:
+        try:
+            address += 1
+        except OSError as e:
+            raise e
+    else:
+        raise

test/simple_source/bug35/01_loop_if_continue.py

@@ -0,0 +1,13 @@
+# From 3.6.4 pathlib.py
+# Bug was handling "continue" as last statement of "if"
+# RUNNABLE!
+def parse_parts(it, parts):
+    for part in it:
+        if not part:
+            continue
+        parts = 1
+    return parts
+
+assert parse_parts([], 5) == 5
+assert parse_parts([True], 6) == 1
+assert parse_parts([False], 6) == 6

test/simple_source/bug35/01_map_unpack.py

@@ -1,9 +1,19 @@
 # Python 3.5+ PEP 448 - Additional Unpacking Generalizations for dictionaries
-{**{}}
-{**{'a': 1, 'b': 2}}
-## {**{'x': 1}, **{'y': 2}}
+# RUNNABLE!
+b = {**{}}
+assert b == {}
+c = {**{'a': 1, 'b': 2}}
+assert c == {'a': 1, 'b': 2}
+d = {**{'x': 1}, **{'y': 2}}
+assert d == {'x': 1, 'y': 2}
 # {'c': 1, {'d': 2}, **{'e': 3}}
 [*[]]
-{**{0:0 for a in b}}
-## {**{}, **{}}
-## {**{}, **{}, **{}}
+
+assert {0: 0} == {**{0:0 for a in c}}
+
+# FIXME: assert deparsing is incorrect for:
+# {**{}, **{}}
+# assert {} == {**{}, **{}, **{}}
+
+# {**{}, **{}, **{}}
+# assert {} == {**{}, **{}, **{}}

test/simple_source/bug35/01_while_if_then.py

@@ -0,0 +1,23 @@
+# From python 3.5.5 telnetlib
+# The bug is the end of a "then" jumping
+# back to the loop which could look like
+# a "continue" and also not like a then/else
+# break
+def process_rawq(self, cmd, cmd2):
+    while self.rawq:
+        if self.iacseq:
+            if cmd:
+                pass
+            elif cmd2:
+                if self.option_callback:
+                    self.option = 2
+                else:
+                    self.option = 3
+
+# From python 3.5.5 telnetlib
+def listener(data):
+    while 1:
+        if data:
+            data = 1
+        else:
+            data = 2

test/simple_source/bug35/04_call_function.py

@@ -1,5 +1,8 @@
 # From sql/schema.py and 3.5 _strptime.py
-# Note that kwargs comes before "positional" args
+# Bug was code not knowing which Python versions
+# have kwargs coming before positional args in code.
+
+# RUNNABLE!
 
 def tometadata(self, metadata, schema, Table, args, name=None):
     table = Table(
@@ -10,3 +13,69 @@ def tometadata(self, metadata, schema, Table, args, name=None):
 
 def _strptime_datetime(cls, args):
     return cls(*args)
+
+
+# From 3.5.5 imaplib
+# Bug is in parsing *date_time[:6] parameter
+from datetime import datetime, timezone, timedelta
+import time
+def Time2Internaldate(date_time):
+    delta = timedelta(seconds=0)
+    return datetime(*date_time[:6], tzinfo=timezone(delta))
+
+assert Time2Internaldate(time.localtime())
+
+# From 3.5.5 tkinter/dialog.py
+def test_varargs0_ext():
+    try:
+        {}.__contains__(*())
+    except TypeError:
+        pass
+
+test_varargs0_ext()
+
+# From 3.4.6 tkinter/dialog.py
+# Bug is in position of *cnf.
+
+def __init__(self, cnf={}):
+    self.num = self.tk.call(
+        'tk_dialog', self._w,
+        cnf['title'], cnf['text'],
+        cnf['bitmap'], cnf['default'],
+        *cnf['strings'])
+
+# From python 3.4.8 multiprocessing/context.py
+def Value(self, fn, typecode_or_type, *args, lock=True):
+    return fn(typecode_or_type, *args, lock=lock,
+              ctx=self.get_context())
+
+# From 3.6.4 heapq.py
+def merge(*iterables, key=None, reverse=False):
+    return
+
+def __call__(self, *args, **kwds):
+    pass
+
+# From 3.6.4 shutil
+def unpack_archive(func, filename, dict, format_info, extract_dir=None):
+    func(filename, extract_dir, **dict(format_info[2]))
+
+# From 3.5.5 test_xrdrlib.py
+import xdrlib
+def assertRaisesConversion(self, *args):
+   self.assertRaises(xdrlib.ConversionError, *args)
+
+# From 3.2.6 _pyio.py
+class BlockingIOError(IOError):
+    def __init__(self, errno, strerror, characters_written=5):
+        super().__init__(errno, strerror)
+
+# From urllib/parse.py
+# Bug was using a subclass made from a call (to namedtuple)
+from collections import namedtuple
+
+class ResultMixin(object):
+    pass
+
+class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin):
+    pass

test/simple_source/bug35/06_try_return.py

@@ -0,0 +1,12 @@
+# From 3.6.4 pdb.py
+# Bug was not having a semantic action for "except_return" tree
+def do_commands(self, arg):
+    if not arg:
+        bnum = 1
+    else:
+        try:
+            bnum = int(arg)
+        except:
+            self.error("Usage:")
+            return
+    self.commands_bnum = bnum

test/simple_source/bug35/07_build_map_unpack.py

@@ -0,0 +1,10 @@
+# Adapted from Python 3.6 trace.py
+# Bug was in handling BUID_TUPLE_UNPACK created via
+# *opts.arguments
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('filename', nargs='?')
+parser.add_argument('arguments', nargs=argparse.REMAINDER)
+opts = parser.parse_args(["foo", "a", "b"])
+argv = opts.filename, *opts.arguments
+assert argv == ('foo', 'a', 'b')

test/simple_source/bug36/01_fstring.py

@@ -1,5 +1,18 @@
+# Self-checking 3.6+ string interpolation tests
+
 var1 = 'x'
 var2 = 'y'
-print(f'interpolate {var1} strings {var2!r} {var2!s} py36')
-print(f'{abc}0')
-print(f'{abc}{abc!s}')
+abc  = 'def'
+assert (f'interpolate {var1} strings {var2!r} {var2!s} py36' ==
+        "interpolate x strings 'y' y py36")
+assert 'def0' == f'{abc}0'
+assert 'defdef' == f'{abc}{abc!s}'
+
+# From 3.6 functools.py
+# Bug was handling format operator strings.
+
+k, v = "1", ["2"]
+x = f"{k}={v!r}"
+y = f"functools.{x}({', '.join(v)})"
+assert x == "1=['2']"
+assert y == "functools.1=['2'](2)"

test/simple_source/bug36/02_kwargs.py

@@ -1,5 +1,13 @@
 # From 3.6 _markupbase _parse_doctype_subset()
-def bug(self, j):
+def bug(self, j, a, b):
     self.parse_comment(j, report=0)
     self.parse_comment(j, report=1, foo=2)
     self.parse_comment(a, b, report=3)
+
+# From 3.6 fnmatch.py
+# Bug was precidence parenthesis around decorator
+
+import functools
+@functools.lru_cache(maxsize=256, typed=True)
+def _compile_pattern(pat):
+    pass

test/simple_source/bug36/03_try_return_except.py

@@ -0,0 +1,30 @@
+# From Python 3.6 bdb.py
+# Bug was handling try's with returns
+# END_FINALLY in 3.6 starts disasppearing
+
+def effective(possibles):
+    for b in possibles:
+        try:
+            return 1
+        except:
+            return 2
+    return 3
+
+assert effective([5]) == 1
+assert effective([]) == 3
+
+def effective2(possibles):
+    b = 0
+    for b in possibles:
+        try:
+            if b >= 5:
+                b = 5
+            else:
+                return 2
+        except:
+            return 3
+    return b
+
+assert effective2([5]) == 5
+assert effective2([]) == 0
+assert effective2(['a']) == 3

test/simple_source/bug36/04_class_kwargs.py

@@ -0,0 +1,17 @@
+# From 3.6 test_abc.py
+# Bug was Reciever() class definition
+import abc
+import unittest
+class TestABCWithInitSubclass(unittest.TestCase):
+    def test_works_with_init_subclass(self):
+        class ReceivesClassKwargs:
+            def __init_subclass__(cls, **kwargs):
+                super().__init_subclass__()
+        class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3):
+            pass
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract

test/simple_source/bug36/04_try_finally.py

@@ -18,3 +18,31 @@ def getvalue1(self):
     finally:
         pass
     return 2
+
+# From Python 3.6 asynchat.py
+# Bug is handling as why in the face of a return.
+# uncompyle6 shows removal of "why" after the return.
+def handle_read(self):
+    try:
+        data = 5
+    except ZeroDivisionError:
+        return
+    except OSError as why:
+        return why
+
+    return data
+
+# From 3.6 contextlib
+# Bug is indentation of "return exc"
+# Also there are extra statements to remove exec,
+# which we hide (unless doing fragments).
+# Note: The indentation bug may be a result of using improper
+# grammar.
+def __exit__(self, type, value, traceback):
+    try:
+        value()
+    except StopIteration as exc:
+        return exc
+    except RuntimeError as exc:
+        return exc
+    return

test/simple_source/bug36/05-for-ifelse.py

@@ -0,0 +1,13 @@
+# From 3.6.4 configparser.py
+# Bug in 3.6 was handling "else" with compound
+# if. there is no POP_BLOCK and
+# there are several COME_FROMs before the else
+def _read(self, fp, a, value, f):
+    for line in fp:
+        for prefix in a:
+            fp()
+        if (value and fp and
+            prefix > 5):
+            f()
+        else:
+            f()

test/simple_source/bug36/05_36lambda.py

@@ -0,0 +1,20 @@
+# From Python 3.6 hmac.py
+# needed to change mklambda rule
+def __init__(self, msg = None, digestmod = None):
+    self.digest_cons = lambda d='': digestmod.new(d)
+
+# From Python 3.6 functools.py
+# Bug was handling lambda for MAKE_FUNCTION_8 (closure)
+# vs to MAKE_FUNCTION_9 (pos_args + closure)
+def bug():
+    def register(cls, func=None):
+        return lambda f: register(cls, f)
+
+# From Python 3.6 configparser.py
+def items(self, d, section=5, raw=False, vars=None):
+    if vars:
+        for key, value in vars.items():
+            d[self.optionxform(key)] = value
+    d = lambda option: self._interpolation.before_get(self,
+        section, option, d[option], d)
+    return

test/simple_source/bug36/05_if_and_comp.py

@@ -0,0 +1,12 @@
+# From 3.6 base64.py
+# Bug was handling "and" condition in the presense of POP_JUMP_IF_FALSE
+# locations
+def _85encode(foldnuls, words):
+    return ['z' if foldnuls and word
+                else  'y'
+                for word in words]
+
+# From Python 3.6 enum.py
+
+def __new__(metacls, cls, bases, classdict):
+    {k: classdict[k] for k in classdict._member_names}

test/simple_source/bug36/05_while_and_if.py

@@ -0,0 +1,13 @@
+# From Python 3.6 getopt.py
+# Bug showing that "while" can have several "COME_FROMS" before loop end
+# NOTE: uncompyle6 still gets the "if"s wrong.
+def getopt(args):
+    while args and args[0] and args[0] != '-':
+        if args[0] == '--':
+            break
+        if args[0]:
+            opts = 5
+        else:
+            opts = 6
+
+    return opts

test/simple_source/bug36/10_argparse.py

@@ -0,0 +1,18 @@
+# From 3.6.4 test_argparse.py
+# Bug was in parsing ** args
+import argparse
+def test_namespace_starkwargs_notidentifier(self):
+    ns = argparse.Namespace(**{'"': 'quote'})
+    string = """Namespace(**{'"': 'quote'})"""
+    assert ns ==  string
+
+def test_namespace_kwargs_and_starkwargs_notidentifier(self):
+    ns = argparse.Namespace(a=1, **{'"': 'quote'})
+    string = """Namespace(a=1, **{'"': 'quote'})"""
+    assert ns == string
+
+
+def test_namespace(self):
+    ns = argparse.Namespace(foo=42, bar='spam')
+    string = "Namespace(bar='spam', foo=42)"
+    assert ns == string

test/simple_source/exception/04_assert_continue.py

@@ -0,0 +1,16 @@
+# Adapted from Python 3.3 idlelib/PyParse.py
+# Bug is continue flowing back to while messing up the determination
+# that it is inside an "if".
+
+# RUNNABLE!
+def _study1(i, n, ch):
+    while i == 3:
+        i = 4
+        if ch:
+            i = 10
+            assert i < 5
+            continue
+        if n:
+            return n
+
+assert _study1(3, 4, False) == 4

test/simple_source/stmts/15_assert.py

@@ -1,3 +1,5 @@
+# RUNNABLE!
+
 # Tests:
 #   2.7:
 #   assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
@@ -16,7 +18,7 @@ for method_name in ['a']:
     if method_name in ('b',):
         method = 'a'
     else:
-        assert 0, "instance installed"
+        assert True, "instance installed"
 
     methods = 'b'
 
@@ -25,5 +27,9 @@ for method_name in ['a']:
 #   if not not do_setlocal:
 #      raise AssertError
 
+# Hmmm.. this isn't strickly a bug
+
 def getpreferredencoding(do_setlocale=True):
     assert not do_setlocale
+
+getpreferredencoding(False)

test/stdlib/.gitignore

@@ -0,0 +1 @@
+/.python-version

test/stdlib/run-and-email.sh

@@ -1,13 +1,20 @@
-#!/usr/bin/bash
+#!/bin/bash
+USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-for VERSION in 2.7.14 2.6.9 ; do
+SUBJECT_PREFIX="stdlib unit testing for"
+for VERSION in 2.6.9 2.7.14 3.4.8 3.5.5 3.6.4 ; do
+    typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
-    ./runtests.sh  >$LOGFILE 2>&1
-    rc=$?
-    if ((rc == 0)); then
-	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+    if ! pyenv local $VERSION ; then
+	rc=1
     else
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+      /bin/bash ./runtests.sh  >$LOGFILE 2>&1
+      rc=$?
+    fi
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
+    else
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" $EMAIL
     fi
 done

test/stdlib/runtests.sh

@@ -48,6 +48,7 @@ case $PYVERSION in
 	;;
     2.6)
 	SKIP_TESTS=(
+	    [test_compile.py]=1  # Intermittent - sometimes works and sometimes doesn't
 	    [test_grp.py]=1      # Long test - might work Control flow?
 	    [test_opcodes.py]=1
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
@@ -77,7 +78,8 @@ case $PYVERSION in
 	    [test_curses.py]=1  # Possibly fails on its own but not detected
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_doctest.py]=1 # Fails on its own
-	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
+	    [test_generators.py]=1  # control flow. uncompyle2 has problem here too
+	    [test_grammar.py]=1     # Too many stmts. Handle large stmts
 	    [test_io.py]=1 # Test takes too long to run
 	    [test_ioctl.py]=1 # Test takes too long to run
 	    [test_itertools.py]=1 # Fix erroneous reduction to "conditional_true".
@@ -96,9 +98,26 @@ case $PYVERSION in
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
         )
 	;;
+    3.5)
+	SKIP_TESTS=(
+	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_quopri.py]=1      # Fails in crontab environment?
+	)
+	;;
+
+    3.6)
+	SKIP_TESTS=(
+	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation
+	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_pow.py]=1         # Control flow wrt "continue"
+	    [test_quopri.py]=1      # Only fails on POWER
+	)
+	;;
     *)
-	SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1
+	SKIP_TESTS=( [test_aepack.py]=1
+		     [audiotests.py]=1
 		     [test_dis.py]=1   # We change line numbers - duh!
+		     [test_generators.py]=1  # I think string formatting of docstrings gets in the way. Not sure
 		   )
 	;;
 esac
@@ -178,7 +197,7 @@ typeset -i ALL_FILES_ENDTIME=$(date +%s)
 
 (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
 
-printf "Ran $i tests in "
+printf "Ran $i unit-test files in "
 displaytime $time_diff
 
 exit $allerrs

uncompyle6/main.py

@@ -38,7 +38,7 @@ def _get_outstream(outfile):
         os.makedirs(dir)
     except OSError:
         pass
-    return open(outfile, 'w')
+    return open(outfile, 'wb')
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
@@ -176,6 +176,7 @@ def main(in_base, out_base, files, codes, outfile=None,
 
     for filename in files:
         infile = os.path.join(in_base, filename)
+        # print("XXX", infile)
         if not os.path.exists(infile):
             sys.stderr.write("File '%s' doesn't exist. Skipped\n"
                              % infile)

uncompyle6/parser.py

@@ -429,9 +429,6 @@ class PythonParser(GenericASTBuilder):
 
         for_block ::= l_stmts_opt _come_froms JUMP_BACK
 
-        for       ::= SETUP_LOOP expr for_iter store
-                      for_block POP_BLOCK _come_froms
-
         forelsestmt ::= SETUP_LOOP expr for_iter store
                         for_block POP_BLOCK else_suite _come_froms
 
@@ -568,9 +565,6 @@ class PythonParser(GenericASTBuilder):
 
         # Positional arguments in make_function
         pos_arg ::= expr
-
-        expr32 ::= expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr
-        expr1024 ::= expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32
         '''
 
     def p_store(self, args):

uncompyle6/parsers/parse2.py

@@ -1,8 +1,21 @@
 #  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#
 #  Copyright (c) 1999 John Aycock
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-A spark grammar for Python 2.x.
+Base grammar for Python 2.x.
 
 However instead of terminal symbols being the usual ASCII text,
 e.g. 5, myvariable, "for", etc.  they are CPython Bytecode tokens,
@@ -80,6 +93,9 @@ class Python2Parser(PythonParser):
         raise_stmt2 ::= expr expr RAISE_VARARGS_2
         raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
 
+        for         ::= SETUP_LOOP expr for_iter store
+                        for_block POP_BLOCK _come_froms
+
         del_stmt ::= expr DELETE_SLICE+0
         del_stmt ::= expr expr DELETE_SLICE+1
         del_stmt ::= expr expr DELETE_SLICE+2
@@ -91,11 +107,8 @@ class Python2Parser(PythonParser):
         _mklambda ::= load_closure mklambda
         kwarg     ::= LOAD_CONST expr
 
-        kvlist ::= kvlist kv3
         kv3 ::= expr expr STORE_MAP
 
-        dict ::= BUILD_MAP kvlist
-
         classdef ::= buildclass store
 
         buildclass ::= LOAD_CONST expr mkfunc
@@ -173,6 +186,7 @@ class Python2Parser(PythonParser):
     def p_expr2(self, args):
         """
         expr ::= LOAD_LOCALS
+        expr ::= LOAD_ASSERT
         expr ::= slice0
         expr ::= slice1
         expr ::= slice2
@@ -279,19 +293,8 @@ class Python2Parser(PythonParser):
             # The order of opname listed is roughly sorted below
             if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
                 v = token.attr
-                thousands = (v//1024)
-                thirty32s = ((v//32) % 32)
-                if thirty32s > 0:
-                    rule = "expr32 ::=%s" % (' expr' * 32)
-                    self.add_unique_rule(rule, opname_base, v, customize)
-                    self.seen32 = True
-                if thousands > 0:
-                    self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),
-                                         opname_base, v, customize)
-                    self.seen1024 = True
                 collection = opname_base[opname_base.find('_')+1:].lower()
-                rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +
-                        'expr32 '*thirty32s + 'expr '*(v % 32) + opname)
+                rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname)
                 self.add_unique_rules([
                     "expr ::= %s" % collection,
                     rule], customize)
@@ -311,11 +314,9 @@ class Python2Parser(PythonParser):
                             'dict_comp_func', 0, customize)
 
                 else:
-                    kvlist_n = "kvlist_%s" % token.attr
-                    self.add_unique_rules([
-                        (kvlist_n + " ::=" + ' kv3' * token.attr),
-                        "dict ::= %s %s" % (opname, kvlist_n)
-                    ], customize)
+                    kvlist_n =  ' kv3' * token.attr
+                    rule = "dict ::= %s%s" % (opname, kvlist_n)
+                    self.addRule(rule, nop_func)
                 continue
             elif opname_base == 'BUILD_SLICE':
                 slice_num  = token.attr
@@ -501,8 +502,9 @@ class Python2Parser(PythonParser):
             self.addRule(rule, nop_func)
             pass
 
-        self.check_reduce['aug_assign1'] = 'AST'
+        self.check_reduce['raise_stmt1'] = 'tokens'
         self.check_reduce['aug_assign2'] = 'AST'
+        self.check_reduce['or'] = 'AST'
         # self.check_reduce['_stmts'] = 'AST'
 
         # Dead code testing...
@@ -518,8 +520,16 @@ class Python2Parser(PythonParser):
         # if lhs == 'while1elsestmt':
         #     from trepan.api import debug; debug()
 
-        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] == 'and':
+        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
             return True
+        elif lhs in ('raise_stmt1',):
+            # We will assme 'LOAD_ASSERT' will be handled by an assert grammar rule
+            return (tokens[first] == 'LOAD_ASSERT' and
+                    (last >= len(tokens) or tokens[last] not in
+                     ('COME_FROM', 'JUMP_BACK','JUMP_FORWARD')))
+        elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
+            expr2 = ast[2]
+            return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
         return False
 
 class Python2ParserSingle(Python2Parser, PythonParserSingle):

uncompyle6/parsers/parse25.py

@@ -73,7 +73,6 @@ class Python25Parser(Python26Parser):
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
         classdefdeco2 ::= LOAD_CONST expr mkfunc CALL_FUNCTION_0 BUILD_CLASS
         kv3 ::= expr expr STORE_MAP
-        kvlist ::= kvlist kv3
         ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
         return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
         return_if_stmt   ::= ret_expr RETURN_END_IF POP_TOP

uncompyle6/parsers/parse26.py

@@ -261,6 +261,9 @@ class Python26Parser(Python2Parser):
 
     def p_misc26(self, args):
         """
+        dict ::= BUILD_MAP kvlist
+        kvlist ::= kvlist kv3
+
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and          ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
 

uncompyle6/parsers/parse27.py

@@ -38,8 +38,6 @@ class Python27Parser(Python2Parser):
         comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
         comp_iter     ::= comp_if
-        comp_iter     ::= comp_if_not
-        comp_if_not   ::= expr jmp_true comp_iter
         comp_iter     ::= comp_body
 
         dict_comp_body ::= expr expr MAP_ADD
@@ -175,6 +173,7 @@ class Python27Parser(Python2Parser):
         """)
         super(Python27Parser, self).customize_grammar_rules(tokens, customize)
         self.check_reduce['and'] = 'AST'
+        self.check_reduce['raise_stmt1'] = 'AST'
         # self.check_reduce['conditional_true'] = 'AST'
         return
 
@@ -191,6 +190,8 @@ class Python27Parser(Python2Parser):
             jmp_target = jmp_false.offset + jmp_false.attr + 3
             return not (jmp_target == tokens[last].offset or
                         tokens[last].pattr == jmp_false.pattr)
+        elif rule[0] == ('raise_stmt1'):
+            return ast[0] == 'expr' and ast[0][0] == 'or'
         # elif rule[0] == ('conditional_true'):
         #     # FIXME: the below is a hack: we check expr for
         #     # nodes that could have possibly been a been a Boolean.

uncompyle6/parsers/parse3.py

@@ -84,8 +84,6 @@ class Python3Parser(PythonParser):
         stmt ::= dict_comp_func
         dict_comp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER store
                            comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
-        dict_comp      ::= LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_0 expr
-                           GET_ITER CALL_FUNCTION_1
 
         comp_iter     ::= comp_if
         comp_iter     ::= comp_if_not
@@ -113,21 +111,25 @@ class Python3Parser(PythonParser):
         continues ::= continue
 
 
-        kwarg   ::= LOAD_CONST expr
-        kwargs  ::= kwarg*
-        kwargs1 ::= kwarg+
+        kwarg      ::= LOAD_CONST expr
+        kwargs     ::= kwarg+
+
 
         classdef ::= build_class store
 
+        # FIXME: we need to add these because don't detect this properly
+        # in custom rules. Specifically if one of the exprs is CALL_FUNCTION
+        # then we'll mistake that for the final CALL_FUNCTION.
+        # We can fix by triggering on the CALL_FUNCTION op
         # Python3 introduced LOAD_BUILD_CLASS
         # Other definitions are in a custom rule
         build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
+        build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
 
         stmt ::= classdefdeco
         classdefdeco ::= classdefdeco1 store
-        classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
-        classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
 
+        expr   ::= LOAD_ASSERT
         assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
 
         assert_expr ::= expr
@@ -241,10 +243,10 @@ class Python3Parser(PythonParser):
         except_suite ::= returns
 
         except_cond1 ::= DUP_TOP expr COMPARE_OP
-                jmp_false POP_TOP POP_TOP POP_TOP
+                         jmp_false POP_TOP POP_TOP POP_TOP
 
         except_cond2 ::= DUP_TOP expr COMPARE_OP
-                jmp_false POP_TOP store POP_TOP
+                         jmp_false POP_TOP store POP_TOP
 
         except  ::=  POP_TOP POP_TOP POP_TOP c_stmts_opt POP_EXCEPT _jump
         except  ::=  POP_TOP POP_TOP POP_TOP returns
@@ -393,9 +395,6 @@ class Python3Parser(PythonParser):
         '''
         load_genexpr ::= LOAD_GENEXPR
         load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_CONST
-
-        # Is there something general going on here?
-        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_CLOSURE_0 expr GET_ITER CALL_FUNCTION_1
         '''
 
     def p_expr3(self, args):
@@ -430,7 +429,7 @@ class Python3Parser(PythonParser):
                         LOAD_CONST CALL_FUNCTION_n
         build_class ::= LOAD_BUILD_CLASS mkfunc
                         expr
-                        call_function
+                        call
                         CALL_FUNCTION_3
          '''
         # FIXME: I bet this can be simplified
@@ -511,7 +510,8 @@ class Python3Parser(PythonParser):
         self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
         if possible_class_decorator:
-            if next_token == 'CALL_FUNCTION' and next_token.attr == 1:
+            if (next_token == 'CALL_FUNCTION' and next_token.attr == 1
+                and args_pos > 1):
                 rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
                         %  (('expr ' * (args_pos-1)), opname, args_pos))
                 self.add_unique_rule(rule, token.kind, uniq_param, customize)
@@ -531,7 +531,7 @@ class Python3Parser(PythonParser):
         subclassing is, well, is pretty base.  And we want it that way: lean and
         mean so that parsing will go faster.
 
-        Here, we add additional grammra rules based on specific instructions
+        Here, we add additional grammar rules based on specific instructions
         that are in the instruction/token stream. In classes that
         inherit from from here and other versions, grammar rules may
         also be removed.
@@ -575,6 +575,11 @@ class Python3Parser(PythonParser):
         seen_LOAD_BUILD_CLASS = False
         seen_GET_AWAITABLE_YIELD_FROM = False
 
+        # This is used in parse36.py as well as here
+        self.seen_LOAD_DICTCOMP = False
+        self.seen_LOAD_SETCOMP = False
+
+
         # Loop over instructions adding custom grammar rules based on
         # a specific instruction seen.
 
@@ -623,13 +628,11 @@ class Python3Parser(PythonParser):
                 self.addRule(rule, nop_func)
             elif opname.startswith('BUILD_LIST_UNPACK'):
                 v = token.attr
-                rule = ('build_list_unpack ::= ' + 'expr1024 ' * int(v//1024) +
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
+                rule = 'build_list_unpack ::= %s%s' % ('expr ' * v, opname)
                 self.addRule(rule, nop_func)
                 rule = 'expr ::= build_list_unpack'
                 self.addRule(rule, nop_func)
-            elif opname_base == 'BUILD_MAP':
+            elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):
                 kvlist_n = "kvlist_%s" % token.attr
                 if opname == 'BUILD_MAP_n':
                     # PyPy sometimes has no count. Sigh.
@@ -644,21 +647,29 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                     rule = "dict ::=  BUILD_MAP_n kvlist_n"
                 elif self.version >= 3.5:
-                    if opname != 'BUILD_MAP_WITH_CALL':
-                        if opname == 'BUILD_MAP_UNPACK':
-                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                    if not opname.startswith('BUILD_MAP_WITH_CALL'):
+                        # FIXME: Use the attr
+                        # so this doesn't run into exponential parsing time.
+                        if opname.startswith('BUILD_MAP_UNPACK'):
                             self.add_unique_rule(rule, opname, token.attr, customize)
                             rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = 'dict ::= ' + 'dict_entry ' * token.attr
-                            self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = ('unmap_dict ::= ' +
-                                    ('dict ' * token.attr) +
-                                    'BUILD_MAP_UNPACK')
+
+                            # FIXME: start here. The LHS should be unmap_dict, not dict.
+                            # FIXME: really we need a combination of dict_entry-like things.
+                            # It just so happens the most common case is not to mix
+                            # dictionary comphensions with dictionary, elements
+                            if self.seen_LOAD_DICTCOMP:
+                                rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)
+                                self.addRule(rule, nop_func)
+                            rule = """
+                             expr       ::= unmap_dict
+                             unmap_dict ::= %s%s
+                             """ % ('expr ' * token.attr, opname)
                         else:
-                            rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
+                            rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
                             self.add_unique_rule(rule, opname, token.attr, customize)
-                            rule = "dict ::=  %s %s" % (kvlist_n, opname)
+                            rule = "dict ::=  %s" % kvlist_n
                 else:
                     rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
                     self.add_unique_rule(rule, opname, token.attr, customize)
@@ -666,15 +677,15 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
                 v = token.attr
-                rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
+                rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)
                 self.addRule(rule, nop_func)
             elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
                 v = token.attr
                 rule = ('starred ::= %s %s' % ('expr ' * v, opname))
                 self.addRule(rule, nop_func)
-            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
+
+            elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',
+                                 'BUILD_TUPLE_UNPACK'):
                 v = token.attr
 
                 is_LOAD_CLOSURE = False
@@ -691,9 +702,7 @@ class Python3Parser(PythonParser):
                         self.add_unique_rule(rule, opname, token.attr, customize)
                 if not is_LOAD_CLOSURE or v == 0:
                     collection = opname_base[opname_base.find('_')+1:].lower()
-                    rule = (('%s ::= ' % collection) + 'expr1024 ' * int(v//1024) +
-                            'expr32 ' * int((v//32) % 32) +
-                            'expr ' * (v % 32) + opname)
+                    rule = '%s ::= %s%s' % (collection, 'expr ' * v, opname)
                     self.add_unique_rules([
                         'expr ::= %s' % collection,
                         rule], customize)
@@ -716,6 +725,19 @@ class Python3Parser(PythonParser):
                                        'CALL_FUNCTION_VAR',
                                        'CALL_FUNCTION_VAR_KW'))
                   or opname.startswith('CALL_FUNCTION_KW')):
+
+                if opname == 'CALL_FUNCTION' and token.attr == 1:
+                    rule = """
+                     dict_comp    ::= LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_0 expr
+                                      GET_ITER CALL_FUNCTION_1
+                    classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
+                    """
+                    if self.version < 3.5:
+                        rule += """
+                        classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
+                        """
+                    self.addRule(rule, nop_func)
+
                 self.custom_classfunc_rule(opname, token, customize,
                                            seen_LOAD_BUILD_CLASS,
                                            seen_GET_AWAITABLE_YIELD_FROM, tokens[i+1])
@@ -784,6 +806,7 @@ class Python3Parser(PythonParser):
                 self.addRule("expr ::= LOAD_CLASSNAME", nop_func)
                 custom_ops_seen.add(opname)
             elif opname == 'LOAD_DICTCOMP':
+                self.seen_LOAD_DICTCOMP = True
                 if has_get_iter_call_function1:
                     rule_pat = ("dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
                                 "GET_ITER CALL_FUNCTION_1")
@@ -798,6 +821,7 @@ class Python3Parser(PythonParser):
             elif opname == 'LOAD_LISTCOMP':
                 self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
             elif opname == 'LOAD_SETCOMP':
+                self.seen_LOAD_SETCOMP = True
                 # Should this be generalized and put under MAKE_FUNCTION?
                 if has_get_iter_call_function1:
                     self.addRule("expr ::= set_comp", nop_func)
@@ -816,6 +840,17 @@ class Python3Parser(PythonParser):
                 # DRY with MAKE_FUNCTION
                 # Note: this probably doesn't handle kwargs proprerly
 
+                if opname == 'MAKE_CLOSURE_0' and self.seen_LOAD_DICTCOMP:
+                    # Is there something general going on here?
+                    # Note that 3.6+ doesn't do this, but we'll remove
+                    # this rule in parse36.py
+                    rule = """
+                        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
+                                      MAKE_CLOSURE_0 expr
+                                      GET_ITER CALL_FUNCTION_1
+                    """
+                    self.addRule(rule, nop_func)
+
                 args_pos, args_kw, annotate_args  = token.attr
 
                 # FIXME: Fold test  into add_make_function_rule
@@ -854,13 +889,13 @@ class Python3Parser(PythonParser):
                                                  opname, token.attr, customize)
 
                 if args_kw > 0:
-                    kwargs_str = 'kwargs1 '
+                    kwargs_str = 'kwargs '
                 else:
                     kwargs_str = ''
 
                 # Note order of kwargs and pos args changed between 3.3-3.4
                 if self.version <= 3.2:
-                    rule = ('mkfunc ::= %s%sload_closure LOAD_CONST kwargs %s'
+                    rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'
                             % (kwargs_str, 'expr ' * args_pos, opname))
                 elif self.version == 3.3:
                     rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'
@@ -870,9 +905,11 @@ class Python3Parser(PythonParser):
                             % ('expr ' * args_pos, kwargs_str, opname))
 
                 self.add_unique_rule(rule, opname, token.attr, customize)
-                rule = ('mkfunc ::= %sload_closure load_genexpr %s'
-                        % ('pos_arg ' * args_pos, opname))
-                self.add_unique_rule(rule, opname, token.attr, customize)
+
+                if args_kw == 0:
+                    rule = ('mkfunc ::= %sload_closure load_genexpr %s'
+                                % ('pos_arg ' * args_pos, opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
 
                 if self.version < 3.4:
                     rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'
@@ -887,6 +924,20 @@ class Python3Parser(PythonParser):
                     # before.
                     args_pos, args_kw, annotate_args, closure  = token.attr
                     stack_count = args_pos + args_kw + annotate_args
+                    if closure:
+                        if args_pos:
+                            rule = ('mklambda ::= %s%s%s%s' %
+                                        ('expr ' * stack_count,
+                                         'load_closure ' * closure,
+                                         'BUILD_TUPLE_1 LOAD_LAMBDA LOAD_CONST ',
+                                        opname))
+                        else:
+                            rule = ('mklambda ::= %s%s%s' %
+                                        ('load_closure ' * closure,
+                                         'LOAD_LAMBDA LOAD_CONST ',
+                                        opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
                     rule = ('mkfunc ::= %s%s%s%s' %
                                 ('expr ' * stack_count,
                                  'load_closure ' * closure,
@@ -954,30 +1005,41 @@ class Python3Parser(PythonParser):
                                 opname))
                     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
+                if args_kw == 0:
+                    kwargs = 'no_kwargs'
+                    self.add_unique_rule("no_kwargs ::=", opname, token.attr, customize)
+                else:
+                    kwargs = 'kwargs'
+
                 if self.version < 3.3:
                     # positional args after keyword args
-                    rule = ('mkfunc ::= kwargs %s%s %s' %
+                    rule = ('mkfunc ::= %s %s%s%s' %
+                            (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST ',
+                             opname))
+                    self.add_unique_rule(rule, opname, token.attr, customize)
+                    rule = ('mkfunc ::= %s%s%s' %
                             ('pos_arg ' * args_pos, 'LOAD_CONST ',
                              opname))
                 elif self.version == 3.3:
                     # positional args after keyword args
-                    rule = ('mkfunc ::= kwargs %s%s %s' %
-                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                    rule = ('mkfunc ::= %s %s%s%s' %
+                            (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST '*2,
                              opname))
                 elif self.version > 3.5:
                     # positional args before keyword args
-                    rule = ('mkfunc ::= %skwargs1 %s %s' %
-                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                    rule = ('mkfunc ::= %s%s %s%s' %
+                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
                              opname))
                 elif self.version > 3.3:
                     # positional args before keyword args
-                    rule = ('mkfunc ::= %skwargs %s %s' %
-                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                    rule = ('mkfunc ::= %s%s %s%s' %
+                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
                              opname))
                 else:
-                    rule = ('mkfunc ::= kwargs %sexpr %s' %
-                            ('pos_arg ' * args_pos, opname))
+                    rule = ('mkfunc ::= %s%sexpr %s' %
+                            (kwargs, 'pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+
                 if opname.startswith('MAKE_FUNCTION_A'):
                     if self.version >= 3.6:
                         rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %
@@ -1077,7 +1139,7 @@ class Python3Parser(PythonParser):
             if tokens[last] in ('JUMP_BACK', 'CONTINUE'):
                 # These indicate inside a loop, but token[last]
                 # should not be in a loop.
-                # FIXME: Not quite righte: refine by using target
+                # FIXME: Not quite right: refine by using target
                 return True
 
             # if SETUP_LOOP target spans the else part, then this is
@@ -1087,7 +1149,7 @@ class Python3Parser(PythonParser):
                 last += 1
             if last == n:
                 return False
-            return tokens[first].attr >= tokens[last].offset
+            return tokens[first].attr > tokens[last].offset
         elif lhs == 'while1stmt':
 
             # If there is a fall through to the COME_FROM_LOOP. then this is

uncompyle6/parsers/parse34.py

@@ -31,19 +31,29 @@ class Python34Parser(Python33Parser):
         expr ::= LOAD_ASSERT
 
 
+        # passtmt is needed for semantic actions to add "pass"
+        suite_stmts_opt ::= pass
+
         # Seems to be needed starting 3.4.4 or so
         while1stmt    ::= SETUP_LOOP l_stmts
                           COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP
+        while1stmt    ::= SETUP_LOOP l_stmts
+                          POP_BLOCK COME_FROM_LOOP
 
         # FIXME the below masks a bug in not detecting COME_FROM_LOOP
         # grammar rules with COME_FROM -> COME_FROM_LOOP already exist
         whileelsestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               else_suitel COME_FROM
 
+        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suitel
+                              COME_FROM_LOOP
+
         # Python 3.4+ optimizes the trailing two JUMPS away
 
         # Is this 3.4 only?
         yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
+
+        _ifstmts_jump ::= c_stmts_opt JUMP_ABSOLUTE JUMP_FORWARD COME_FROM
         """
 
     def customize_grammar_rules(self, tokens, customize):