Get ready for release 3.1.2

which are created via a call to create a subclass.

Should be more general though.

.gitignore

@@ -21,3 +21,4 @@
 ChangeLog
 __pycache__
 build
+nohup.out

.travis.yml

@@ -4,7 +4,7 @@ sudo: false
 
 python:
   - '3.5'
-  - '2.7.12'
+  - '2.7'
   - '2.6'
   - '3.3'
   - '3.4'

Makefile

@@ -27,21 +27,18 @@ check:
 check-short: pytest
 	$(MAKE) -C test check-short
 
+# Note for 2.6 use <=3.0.1 see requirements-dev.txt
 #: Tests for Python 2.7, 3.3 and 3.4
-check-2.7 check-3.3 check-3.4: pytest
+check-2.6 check-2.7 check-3.3 check-3.4 check-3.5: pytest
 	$(MAKE) -C test $@
 
 #: Tests for Python 3.2 and 3.5 - pytest doesn't work here
 # Or rather 3.5 doesn't work not on Travis
-check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
+check-3.0 check-3.1 check-3.2 check-3.6:
 	$(MAKE) -C test $@
 
 check-3.7: pytest
 
-#:Tests for Python 2.6 (doesn't have pytest)
-check-2.6:
-	$(MAKE) -C test $@
-
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:

NEWS

@@ -1,3 +1,32 @@
+uncompyle6 3.1.2 2018-04-08 Eastern Orthodox Easter
+
+- Python 3.x subclass and call parsing fixes
+- Allow/note running on Python 3.1
+- improve 3.5+ BUILD_MAP_UNPACK
+- DRY instruction building code between 2.x and 3.x
+- expand testing
+
+uncompyle6 3.1.1 2018-04-01 Easter April Fool's
+
+Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
+
+- fill out 3.5+ BUILD_MAP_UNPACK (more work is needed)
+- fill out 3.4+ CALL_FUNCTION_... (more work is needed)
+- fill out 3.5 MAKE_FUNCTION  (more work is needed)
+- reduce 3.5, 3.6 control-flow bugs
+- reduce ambiguity in rules that lead to long (exponential?) parses
+- limit/isolate some 2.6/2.7,3.x grammar rules
+- more runtime testing of decompiled code
+- more removal of parenthesis around calls via setting precidence
+
+uncompyle6 3.1.0 2018-03-21 Equinox
+
+- Add code_deparse_with_offset() fragment function.
+- Correct paramenter call fragment deparse_code()
+- Lots of 3.6, 3.x, and 2.7 bug fixes
+  About 5% of 3.6 fail parsing now. But
+  semantics still needs much to be desired.
+
 uncompyle6 3.0.1 2018-02-17
 
 - All Python 2.6.9 standard library files weakly verify

__pkginfo__.py

@@ -56,7 +56,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.7.0, < 3.8.0', 'six']
+                      'xdis >= 3.8.0, < 3.9.0', 'six']
 
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.4.8'
+export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'

pytest/test_fjt.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
-from xdis.bytecode import Bytecode
-from array import array
 def bug(state, slotstate):
     if state:
         if slotstate is not None:
@@ -24,16 +22,8 @@ def bug_loop(disassemble, tb=None):
 def test_if_in_for():
     code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
-    print(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
-        scan.build_prev_op(n)
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
@@ -48,14 +38,7 @@ def test_if_in_for():
         #    {'start': 62, 'end': 63, 'type': 'for-else'}]
 
         code = bug_loop.__code__
-        n = scan.setup_code(code)
-        bytecode = Bytecode(code, scan.opc)
-        scan.build_lines_data(code, n)
-        scan.insts = list(bytecode)
-        scan.build_prev_op(n)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt = scan.find_jump_targets(False)
         assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
         assert scan.structs == [
@@ -69,14 +52,7 @@ def test_if_in_for():
             {'start': 48, 'end': 67, 'type': 'while-loop'}]
 
     elif 3.2 < PYTHON_VERSION <= 3.4:
-        bytecode = Bytecode(code, scan.opc)
-        scan.code = array('B', code.co_code)
-        scan.lines = scan.build_lines_data(code)
-        scan.build_prev_op()
-        scan.insts = list(bytecode)
-        scan.offset2inst_index = {}
-        for i, inst in enumerate(scan.insts):
-            scan.offset2inst_index[inst.offset] = i
+        scan.build_instructions(code)
         fjt  = scan.find_jump_targets(False)
         assert {69: [66], 63: [18]} == fjt
         assert scan.structs == \
@@ -86,5 +62,6 @@ def test_if_in_for():
            {'end': 59, 'type': 'for-loop', 'start': 31},
            {'end': 63, 'type': 'for-else', 'start': 62}]
     else:
-        assert True, "FIXME: should note fixed"
+        print("FIXME: should fix for %s" % PYTHON_VERSION)
+        assert True
     return

pytest/test_function_call.py

@@ -6,7 +6,7 @@ import pytest
 # uncompyle
 from validate import validate_uncompyle
 from test_fstring import expressions
-
+from uncompyle6 import PYTHON_VERSION
 
 alpha = st.sampled_from(string.ascii_lowercase)
 numbers = st.sampled_from(string.digits)
@@ -81,7 +81,8 @@ def function_calls(draw,
 def test_function_no_args():
     validate_uncompyle("fn()")
 
-
+@pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                    reason="need at least Python 2.7")
 def isolated_function_calls(which):
     """
     Returns a strategy for generating function calls, but isolated to
@@ -108,21 +109,29 @@ def isolated_function_calls(which):
 
 with settings(max_examples=25):
 
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
     @given(isolated_function_calls('positional'))
     @example("fn(0)")
     def test_function_positional_only(expr):
         validate_uncompyle(expr)
 
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
     @given(isolated_function_calls('keyword'))
     @example("fn(a=0)")
     def test_function_call_keyword_only(expr):
         validate_uncompyle(expr)
 
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
     @given(isolated_function_calls('star'))
     @example("fn(*items)")
     def test_function_call_star_only(expr):
         validate_uncompyle(expr)
 
+    @pytest.mark.skipif(PYTHON_VERSION < 2.7,
+                        reason="need at least Python 2.7")
     @given(isolated_function_calls('double_star'))
     @example("fn(**{})")
     def test_function_call_double_star_only(expr):

pytest/test_grammar.py

@@ -18,40 +18,44 @@ def test_grammar():
      right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['expr1024', 'pos_arg', 'get_iter', 'attribute'])
+    expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
     unused_rhs = set(['list', 'mkfunc',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
-    if PYTHON3:
-        expect_lhs.add('load_genexpr')
+
+    if PYTHON_VERSION > 2.6:
         expect_lhs.add('kvlist')
         expect_lhs.add('kv3')
+        unused_rhs.add('dict')
+
+    if PYTHON3:
+        expect_lhs.add('load_genexpr')
 
         unused_rhs = unused_rhs.union(set("""
-        except_pop_except generator_exp classdefdeco2
-        dict
+        except_pop_except generator_exp
         """.split()))
         if PYTHON_VERSION >= 3.0:
             expect_lhs.add("annotate_arg")
             expect_lhs.add("annotate_tuple")
             unused_rhs.add("mkfunc_annotate")
             unused_rhs.add('call')
+            unused_rhs.add("dict_comp")
+            unused_rhs.add("classdefdeco1")
             if PYTHON_VERSION < 3.6:
                 # 3.6 has at least one non-custom call rule
                 # the others don't
                 unused_rhs.add('call')
                 if PYTHON_VERSION == 3.5:
                     expect_right_recursive.add((('l_stmts',
-                                                 ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                                                 ('lastl_stmt', 'come_froms', 'l_stmts'))))
                     pass
                 pass
             else:
                 expect_right_recursive.add((('l_stmts',
                                              ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
-                # expect_lhs.add('kwargs1')
                 pass
             pass
         pass
@@ -85,6 +89,8 @@ def test_grammar():
             """.split())
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
+        if PYTHON_VERSION == 2.6:
+            opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
         ignore_set.add('LOAD_CLASSNAME')

pytest/test_single_compile.py

@@ -1,6 +1,8 @@
 import pytest
-from uncompyle6 import PYTHON_VERSION, PYTHON3, deparse_code
+from uncompyle6 import PYTHON_VERSION, deparse_code
 
+@pytest.mark.skip(PYTHON_VERSION < 2.7,
+                  reason="need at least Python 2.7")
 def test_single_mode():
     single_expressions = (
         'i = 1',

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest>=3.0.0
+pytest>=3.0.0,<=3.0.1
 flake8
-hypothesis<=3.8.3
+hypothesis<=3.0.0

setup.py

@@ -4,12 +4,12 @@ import sys
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 7)) or ((3, 0) <= SYS_VERSION <= (3, 1)):
-    mess = "Python Release 2.6 .. 3.7 excluding 3.0 and 3.1 are supported in this code branch."
+if not ((2, 6) <= SYS_VERSION  <= (3, 7)) or ((3, 0) <= SYS_VERSION <= (3, 0)):
+    mess = "Python Release 2.6 .. 3.7 excluding 3.0 are supported in this code branch."
     if ((2, 4) <= SYS_VERSION <= (2, 7)):
         mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
                  sys.version[0:3])
-    elif SYS_VERSION < (2, 4) or ((3, 0) <= SYS_VERSION <= (3, 1)):
+    elif SYS_VERSION < (2, 4) or (3, 0) <= SYS_VERSION:
         mess += ("\nThis package is not supported for Python version %s."
                  % sys.version[0:3])
     print(mess)

test/.gitignore

@@ -0,0 +1 @@
+/nohup.out

test/Makefile

@@ -3,9 +3,9 @@ PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clea
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-5.6 5.6 5.8 \
-      grammar-coverage-2.5 grammar-coverage-2.6 grammarcoverage-2.7 \
-      grammar-coverage-3.1 grammar-coverage-3.2 grammarcoverage-3.3 \
-      grammar-coverage-3.4 grammar-coverage-3.5 grammarcoverage-3.6
+      grammar-coverage-2.5 grammar-coverage-2.6 grammar-coverage-2.7 \
+      grammar-coverage-3.1 grammar-coverage-3.2 grammar-coverage-3.3 \
+      grammar-coverage-3.4 grammar-coverage-3.5 grammar-coverage-3.6
 
 
 GIT2CL ?= git2cl
@@ -37,18 +37,22 @@ check-3.0: check-bytecode
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
 	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
@@ -114,23 +118,29 @@ check-bytecode-2.4:
 check-bytecode-2.5:
 	$(PYTHON) test_pythonlib.py --bytecode-2.5
 
+#: Get grammar coverage for Python 2.4
+grammar-coverage-2.4:
+	-rm $(COVER_DIR)/spark-grammar-24.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.4.cover $(PYTHON) test_pyenvlib.py --2.4.6 --max= 800
+
 #: Get grammar coverage for Python 2.5
 grammar-coverage-2.5:
-	-rm $(COVER_DIR)/spark-grammar-25.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
+	-rm $(COVER_DIR)/spark-grammar-2.5.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.5.cover $(PYTHON) test_pyenvlib.py --2.5.6 --max=800
 
 #: Get grammar coverage for Python 2.6
 grammar-coverage-2.6:
-	-rm $(COVER_DIR)/spark-grammar-26.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
+	-rm $(COVER_DIR)/spark-grammar-2.6.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.6.cover $(PYTHON) test_pyenvlib.py --2.6.9 --max=800
 
 #: Get grammar coverage for Python 2.7
 grammar-coverage-2.7:
-	-rm $(COVER_DIR)/spark-grammar-27.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
+	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:
@@ -141,33 +151,39 @@ SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-30.cover $(PYTHON) test_pythonl
 
 #: Get grammar coverage for Python 3.1
 grammar-coverage-3.1:
-	-rm $(COVER_DIR)/spark-grammar-31.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-31.cover $(PYTHON) test_pyenvlib.py --3.1.5
+	-rm $(COVER_DIR)/spark-grammar-3.1.cover
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pythonlib.py --bytecode-3.1
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.1.cover $(PYTHON) test_pyenvlib.py --3.1.5
 
 #: Get grammar coverage for Python 3.2
 grammar-coverage-3.2:
-	-rm $(COVER_DIR)/spark-grammar-32.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-32.cover $(PYTHON) test_pyenvlib.py --3.2.6
+	-rm $(COVER_DIR)/spark-grammar-3.2.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pythonlib.py --bytecode-3.2
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.2.cover $(PYTHON) test_pyenvlib.py --3.2.6
 
 #: Get grammar coverage for Python 3.3
 grammar-coverage-3.3:
-	-rm $(COVER_DIR)/spark-grammar-33.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-33.cover $(PYTHON) test_pyenvlib.py --3.3.6
+	-rm $(COVER_DIR)/spark-grammar-3.3.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pythonlib.py --bytecode-3.3
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.3.cover $(PYTHON) test_pyenvlib.py --3.3.7 --max=800
 
 #: Get grammar coverage for Python 3.4
 grammar-coverage-3.4:
-	-rm $(COVER_DIR)/spark-grammar-34.cover
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-34.cover $(PYTHON) test_pyenvlib.py --3.4.2
+	-rm $(COVER_DIR)/spark-grammar-3.4.cover || true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pythonlib.py --bytecode-3.4
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.4.cover $(PYTHON) test_pyenvlib.py --3.4.8 --max=800
 
 #: Get grammar coverage for Python 3.5
 grammar-coverage-3.5:
-	rm $(COVER_DIR)/spark-grammar-35.cover || /bin/true
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-35.cover $(PYTHON) test_pyenvlib.py --3.5.3
+	rm $(COVER_DIR)/spark-grammar-3.5.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pythonlib.py --bytecode-3.5
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.5.cover $(PYTHON) test_pyenvlib.py --3.5.5 --max=450
+
+#: Get grammar coverage for Python 3.6
+grammar-coverage-3.6:
+	rm $(COVER_DIR)/spark-grammar-3.6.cover || /bin/true
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pythonlib.py --bytecode-3.6
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-3.6.cover $(PYTHON) test_pyenvlib.py --3.6.4 --max=280
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
@@ -186,10 +202,12 @@ check-bytecode-3.0:
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
+	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:

test/grammar-cover/.gitignore

@@ -0,0 +1 @@
+/.python-version

test/grammar-cover/README.md

@@ -0,0 +1 @@
+Code in this directory gets statistics on grammar coverage

test/grammar-cover/convert.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+for VERS in 2{4,5,6,7} 3{2,3,4,5} ; do
+    GRAMMAR_TXT=grammar-${VERS}.txt
+    spark-parser-coverage --max-count 3000 --path spark-grammar-${VERS}.cover > $GRAMMAR_TXT
+done

test/grammar-cover/grammar-all.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+$SHELL ./grammar.sh 2.4 2.5 2.6 2.7 3.2 3.3 3.4 3.5 3.6

test/grammar-cover/grammar.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Remake Python grammar statistics
+
+typeset -A ALL_VERS=([2.4]=2.4.6 [2.5]=2.5.6 [2.6]=2.6.9 [2.7]=2.7.14 [3.2]=3.2.6 [3.3]=3.3.6 [3.4]=3.4.8 [3.5]=3.5.5 [3.6]=3.6.4)
+
+if (( $# == 0 )); then
+    echo 1>&2 "usage: $0 two-digit-version"
+    exit 1
+fi
+
+me=${BASH_SOURCE[0]}
+workdir=$(dirname $me)
+cd $workdir
+workdir=$(pwd)
+while [[ -n $1 ]]  ; do
+    SHORT_VERSION=$1; shift
+    LONG_VERSION=${ALL_VERS[$SHORT_VERSION]}
+    if [[ -z ${LONG_VERSION} ]] ; then
+       echo 1>&2 "Version $SHORT_VERSION not known"
+       exit 2
+    fi
+
+    tmpdir=$workdir/../../tmp/grammar-cover
+    COVER_FILE=${tmpdir}/spark-grammar-${SHORT_VERSION}.cover
+    [[ -d  $tmpdir ]] || mkdir $tmpdir
+    cd $workdir/../..
+    if [[ $SHORT_VERSION > 2.5 ]] ; then
+        source ./admin-tools/setup-master.sh
+    else
+        source ./admin-tools/setup-python-2.4.sh
+    fi
+    GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
+    (cd ../.. && pyenv local ${LONG_VERSION})
+    cd ./test
+    if [[ -r $COVER_FILE ]]; then
+	rm $COVER_FILE
+    fi
+    if [[ -r $GRAMMAR_TXT ]]; then
+        GRAMMAR_SAVE_TXT=${tmpdir}/grammar-${SHORT_VERSION}-save.txt
+        cp $GRAMMAR_TXT $GRAMMAR_SAVE_TXT
+    fi
+    make grammar-coverage-${SHORT_VERSION};
+    spark-parser-coverage --max-count=3000 --path $COVER_FILE > $GRAMMAR_TXT
+done

test/grammar-cover/run-and-email.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+USER=${USER:-rocky}
+EMAIL=${EMAIL:-rb@dustyfeet.com}
+SUBJECT_PREFIX="grammar cover testing for"
+LOGFILE=/tmp/grammar-cover-$$.log
+/bin/bash ./grammar-all.sh  >$LOGFILE 2>&1
+rc=$?
+if ((rc == 0)); then
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX ok" ${USER}@localhost
+else
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX not ok" ${USER}@localhost
+    tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX not ok" $EMAIL
+fi

test/run-and-email.sh

@@ -1,13 +1,69 @@
-#!/usr/bin/bash
+#!/bin/bash
+
+function displaytime {
+    printf "ran in "
+    local T=$1
+    local D=$((T/60/60/24))
+    local H=$((T/60/60%24))
+    local M=$((T/60%60))
+    local S=$((T%60))
+    (( $D > 0 )) && printf '%d days ' $D
+    (( $H > 0 )) && printf '%d hours ' $H
+    (( $M > 0 )) && printf '%d minutes ' $M
+    (( $D > 0 || $H > 0 || $M > 0 )) && printf 'and '
+    printf '%d seconds\n' $S
+}
+
+PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.3.7 3.4.8 2.6.9 3.6.4"}
+# PYVERSION=${PYVERSION:-"3.5.5"}
+
+USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-for VERSION in 2.7.14 2.6.9 ; do
-    LOGFILE=/tmp/pyenlib-$VERSION-$$.log
-    python ./test_pyenvlib.py --max 800 --weak-verify --$VERSION  >$LOGFILE 2>&1
-    rc=$?
-    if ((rc == 0)); then
-	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+MAX_TESTS=${MAX_TESTS:-800}
+typeset -i RUN_STARTTIME=$(date +%s)
+
+for VERSION in $PYVERSION ; do
+    typeset -i rc=0
+    LOGFILE=/tmp/pyenvlib-$VERSION-$$.log
+
+    if [[ $VERSION == '3.5.5' ]] ; then
+	MAX_TESTS=224
+    elif [[ $VERSION == '3.2.6' ]] ; then
+	MAX_TESTS=700
+    elif [[ $VERSION == '3.6.4' ]] ; then
+	MAX_TESTS=400
     else
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+	MAX_TESTS=800
     fi
+
+    if ! pyenv local $VERSION ; then
+	rc=1
+    else
+      echo Python Version $(pyenv local) > $LOGFILE
+      echo "" >> $LOGFILE
+      typeset -i ALL_FILES_STARTTIME=$(date +%s)
+      python ./test_pyenvlib.py --max ${MAX_TESTS} --weak-verify --$VERSION  >>$LOGFILE 2>&1
+      rc=$?
+
+      echo Python Version $(pyenv local) >> $LOGFILE
+      echo "" >>$LOGFILE
+
+      typeset -i ALL_FILES_ENDTIME=$(date +%s)
+      (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
+      displaytime $time_diff >> $LOGFILE
+    fi
+
+    SUBJECT_PREFIX="pyenv weak verify (max $MAX_TESTS) for"
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
+    else
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${EMAIL}
+    fi
+    rm .python-version
 done
+
+typeset -i RUN_ENDTIME=$(date +%s)
+(( time_diff =  RUN_ENDTIME - RUN_STARTTIME))
+elapsed_time=$(displaytime $time_diff)
+echo "Run complete $elapsed_time for versions $PYVERSION" | mail -s "pyenv weak verify in $elapsed_time" ${EMAIL}

test/simple_source/bug25/04_try_except_else.py

@@ -0,0 +1,18 @@
+# Bug found in 2.4 test_math.py
+# Bug was turning last try/except/else into try/else
+import math
+def test_exceptions():
+    try:
+        x = math.exp(-1000000000)
+    except:
+        raise RuntimeError
+
+    x = 1
+    try:
+        x = math.sqrt(-1.0)
+    except ValueError:
+        return x
+    else:
+        raise RuntimeError
+
+test_exceptions()

test/simple_source/bug27+/02_assert.py

@@ -0,0 +1,31 @@
+# From 2.7 test_argparse.py
+# Bug was turning assert into an "or raise" statement
+def __call__(arg, dest):
+    try:
+        assert arg == 'spam', 'dest: %s' % dest
+    except:
+        raise
+
+__call__('spam', __file__)
+
+# From python 2.7.14 lib2to3/refactor.py
+# Bug was mangling assert turning if into "or"
+def refactor_doctest(clipped, new):
+    assert clipped, clipped
+    if not new:
+        new += u"\n"
+    return
+
+# From 2.7.14 test_hashlib.py
+# The bug was turning assert into an "if"
+# statement which isn't wrong, but we got the
+# range of the if incorrect. When we have
+# better control flow analysis we can revisit.
+def test_threaded_hashing():
+    for threadnum in xrange(1):
+        result = 1
+        assert result > 0
+        result = 2
+    return result
+
+assert test_threaded_hashing() == 2

test/simple_source/bug27+/05_not_unconditional.py

@@ -1,5 +1,5 @@
 # From 2.7 test_itertools.py
-# Bug was in 2.7 decompiling like the commented out
+# Bug was in 2.7 decompiling the target assignment
 # code below
 from itertools import izip_longest
 for args in [

test/simple_source/bug31/05_abc_test.py

@@ -0,0 +1,18 @@
+# Extracted from Python 3.5 test_abc.py
+# Bug is class having only a single kwarg
+# subclass.
+import abc
+import unittest
+from inspect import isabstract
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract
+            def bar(self): pass  # concrete
+        assert C.__abstractmethods__, {"foo"}
+        assert isabstract(C)
+        pass
+
+test_abstractmethod_integration(None)

test/simple_source/bug33/02_named_and_kwargs.py

@@ -1,5 +1,6 @@
 # Python 3.6 subprocess.py bug
 # Bug is getting params correct: timeout before **kwargs
+import subprocess
 def call(*popenargs, timeout=None, **kwargs):
     return
 
@@ -14,6 +15,9 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
 # From 3.4 asyncio/locks.py
 # Bug was handling" "value=1, *"
 
+class Semaphore:
+    pass
+
 class BoundedSemaphore(Semaphore):
     def __init__(self, value=1, *, loop=None):
         super().__init__(value, loop=loop)

test/simple_source/bug33/05_nonlocal.py

@@ -0,0 +1,12 @@
+# From Python 3.6 functools.py
+# Bug was in detecting "nonlocal" access
+def not_bug():
+    cache_token = 5
+
+    def register():
+        nonlocal cache_token
+        return cache_token == 5
+
+    return register()
+
+assert not_bug()

test/simple_source/bug34/05_while1_if_continue.py

@@ -0,0 +1,37 @@
+# Bug in Python 3.4 text_file.py
+# Bug is handling:  while true ... if ... continue
+def readline(b):
+    a = 1
+    while True:
+        if b:
+            if b[0]:
+                a = 2
+                b = None
+                continue
+        b = None
+        a = 5
+
+        return a
+
+assert readline(None) == 1
+assert readline([2]) == 2
+
+def readline2(self):
+    while True:
+        line = 5
+        if self[0]:
+            if self:
+                self[0] = 1
+                continue
+
+        return line + self[0]
+
+# From 3.4.4 connection.py
+def PipeClient(address):
+    while 1:
+        try:
+            address += 1
+        except OSError as e:
+            raise e
+    else:
+        raise

test/simple_source/bug35/01_loop_if_continue.py

@@ -0,0 +1,13 @@
+# From 3.6.4 pathlib.py
+# Bug was handling "continue" as last statement of "if"
+# RUNNABLE!
+def parse_parts(it, parts):
+    for part in it:
+        if not part:
+            continue
+        parts = 1
+    return parts
+
+assert parse_parts([], 5) == 5
+assert parse_parts([True], 6) == 1
+assert parse_parts([False], 6) == 6

test/simple_source/bug35/01_map_unpack.py

@@ -1,9 +1,19 @@
 # Python 3.5+ PEP 448 - Additional Unpacking Generalizations for dictionaries
-{**{}}
-{**{'a': 1, 'b': 2}}
-## {**{'x': 1}, **{'y': 2}}
+# RUNNABLE!
+b = {**{}}
+assert b == {}
+c = {**{'a': 1, 'b': 2}}
+assert c == {'a': 1, 'b': 2}
+d = {**{'x': 1}, **{'y': 2}}
+assert d == {'x': 1, 'y': 2}
 # {'c': 1, {'d': 2}, **{'e': 3}}
 [*[]]
-{**{0:0 for a in b}}
-## {**{}, **{}}
-## {**{}, **{}, **{}}
+
+assert {0: 0} == {**{0:0 for a in c}}
+
+# FIXME: assert deparsing is incorrect for:
+# {**{}, **{}}
+# assert {} == {**{}, **{}, **{}}
+
+# {**{}, **{}, **{}}
+# assert {} == {**{}, **{}, **{}}

test/simple_source/bug35/01_while_if_then.py

@@ -0,0 +1,23 @@
+# From python 3.5.5 telnetlib
+# The bug is the end of a "then" jumping
+# back to the loop which could look like
+# a "continue" and also not like a then/else
+# break
+def process_rawq(self, cmd, cmd2):
+    while self.rawq:
+        if self.iacseq:
+            if cmd:
+                pass
+            elif cmd2:
+                if self.option_callback:
+                    self.option = 2
+                else:
+                    self.option = 3
+
+# From python 3.5.5 telnetlib
+def listener(data):
+    while 1:
+        if data:
+            data = 1
+        else:
+            data = 2

test/simple_source/bug35/04_call_function.py

@@ -1,5 +1,8 @@
 # From sql/schema.py and 3.5 _strptime.py
-# Note that kwargs comes before "positional" args
+# Bug was code not knowing which Python versions
+# have kwargs coming before positional args in code.
+
+# RUNNABLE!
 
 def tometadata(self, metadata, schema, Table, args, name=None):
     table = Table(
@@ -10,3 +13,69 @@ def tometadata(self, metadata, schema, Table, args, name=None):
 
 def _strptime_datetime(cls, args):
     return cls(*args)
+
+
+# From 3.5.5 imaplib
+# Bug is in parsing *date_time[:6] parameter
+from datetime import datetime, timezone, timedelta
+import time
+def Time2Internaldate(date_time):
+    delta = timedelta(seconds=0)
+    return datetime(*date_time[:6], tzinfo=timezone(delta))
+
+assert Time2Internaldate(time.localtime())
+
+# From 3.5.5 tkinter/dialog.py
+def test_varargs0_ext():
+    try:
+        {}.__contains__(*())
+    except TypeError:
+        pass
+
+test_varargs0_ext()
+
+# From 3.4.6 tkinter/dialog.py
+# Bug is in position of *cnf.
+
+def __init__(self, cnf={}):
+    self.num = self.tk.call(
+        'tk_dialog', self._w,
+        cnf['title'], cnf['text'],
+        cnf['bitmap'], cnf['default'],
+        *cnf['strings'])
+
+# From python 3.4.8 multiprocessing/context.py
+def Value(self, fn, typecode_or_type, *args, lock=True):
+    return fn(typecode_or_type, *args, lock=lock,
+              ctx=self.get_context())
+
+# From 3.6.4 heapq.py
+def merge(*iterables, key=None, reverse=False):
+    return
+
+def __call__(self, *args, **kwds):
+    pass
+
+# From 3.6.4 shutil
+def unpack_archive(func, filename, dict, format_info, extract_dir=None):
+    func(filename, extract_dir, **dict(format_info[2]))
+
+# From 3.5.5 test_xrdrlib.py
+import xdrlib
+def assertRaisesConversion(self, *args):
+   self.assertRaises(xdrlib.ConversionError, *args)
+
+# From 3.2.6 _pyio.py
+class BlockingIOError(IOError):
+    def __init__(self, errno, strerror, characters_written=5):
+        super().__init__(errno, strerror)
+
+# From urllib/parse.py
+# Bug was using a subclass made from a call (to namedtuple)
+from collections import namedtuple
+
+class ResultMixin(object):
+    pass
+
+class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin):
+    pass

test/simple_source/bug35/06_try_return.py

@@ -0,0 +1,12 @@
+# From 3.6.4 pdb.py
+# Bug was not having a semantic action for "except_return" tree
+def do_commands(self, arg):
+    if not arg:
+        bnum = 1
+    else:
+        try:
+            bnum = int(arg)
+        except:
+            self.error("Usage:")
+            return
+    self.commands_bnum = bnum

test/simple_source/bug35/07_build_map_unpack.py

@@ -0,0 +1,10 @@
+# Adapted from Python 3.6 trace.py
+# Bug was in handling BUID_TUPLE_UNPACK created via
+# *opts.arguments
+import argparse
+parser = argparse.ArgumentParser()
+parser.add_argument('filename', nargs='?')
+parser.add_argument('arguments', nargs=argparse.REMAINDER)
+opts = parser.parse_args(["foo", "a", "b"])
+argv = opts.filename, *opts.arguments
+assert argv == ('foo', 'a', 'b')

test/simple_source/bug36/01_fstring.py

@@ -1,5 +1,18 @@
+# Self-checking 3.6+ string interpolation tests
+
 var1 = 'x'
 var2 = 'y'
-print(f'interpolate {var1} strings {var2!r} {var2!s} py36')
-print(f'{abc}0')
-print(f'{abc}{abc!s}')
+abc  = 'def'
+assert (f'interpolate {var1} strings {var2!r} {var2!s} py36' ==
+        "interpolate x strings 'y' y py36")
+assert 'def0' == f'{abc}0'
+assert 'defdef' == f'{abc}{abc!s}'
+
+# From 3.6 functools.py
+# Bug was handling format operator strings.
+
+k, v = "1", ["2"]
+x = f"{k}={v!r}"
+y = f"functools.{x}({', '.join(v)})"
+assert x == "1=['2']"
+assert y == "functools.1=['2'](2)"

test/simple_source/bug36/02_kwargs.py

@@ -1,5 +1,13 @@
 # From 3.6 _markupbase _parse_doctype_subset()
-def bug(self, j):
+def bug(self, j, a, b):
     self.parse_comment(j, report=0)
     self.parse_comment(j, report=1, foo=2)
     self.parse_comment(a, b, report=3)
+
+# From 3.6 fnmatch.py
+# Bug was precidence parenthesis around decorator
+
+import functools
+@functools.lru_cache(maxsize=256, typed=True)
+def _compile_pattern(pat):
+    pass

test/simple_source/bug36/03_try_return_except.py

@@ -0,0 +1,30 @@
+# From Python 3.6 bdb.py
+# Bug was handling try's with returns
+# END_FINALLY in 3.6 starts disasppearing
+
+def effective(possibles):
+    for b in possibles:
+        try:
+            return 1
+        except:
+            return 2
+    return 3
+
+assert effective([5]) == 1
+assert effective([]) == 3
+
+def effective2(possibles):
+    b = 0
+    for b in possibles:
+        try:
+            if b >= 5:
+                b = 5
+            else:
+                return 2
+        except:
+            return 3
+    return b
+
+assert effective2([5]) == 5
+assert effective2([]) == 0
+assert effective2(['a']) == 3

test/simple_source/bug36/04_class_kwargs.py

@@ -0,0 +1,17 @@
+# From 3.6 test_abc.py
+# Bug was Reciever() class definition
+import abc
+import unittest
+class TestABCWithInitSubclass(unittest.TestCase):
+    def test_works_with_init_subclass(self):
+        class ReceivesClassKwargs:
+            def __init_subclass__(cls, **kwargs):
+                super().__init_subclass__()
+        class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3):
+            pass
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract

test/simple_source/bug36/04_try_finally.py

@@ -18,3 +18,31 @@ def getvalue1(self):
     finally:
         pass
     return 2
+
+# From Python 3.6 asynchat.py
+# Bug is handling as why in the face of a return.
+# uncompyle6 shows removal of "why" after the return.
+def handle_read(self):
+    try:
+        data = 5
+    except ZeroDivisionError:
+        return
+    except OSError as why:
+        return why
+
+    return data
+
+# From 3.6 contextlib
+# Bug is indentation of "return exc"
+# Also there are extra statements to remove exec,
+# which we hide (unless doing fragments).
+# Note: The indentation bug may be a result of using improper
+# grammar.
+def __exit__(self, type, value, traceback):
+    try:
+        value()
+    except StopIteration as exc:
+        return exc
+    except RuntimeError as exc:
+        return exc
+    return

test/simple_source/bug36/05-for-ifelse.py

@@ -0,0 +1,13 @@
+# From 3.6.4 configparser.py
+# Bug in 3.6 was handling "else" with compound
+# if. there is no POP_BLOCK and
+# there are several COME_FROMs before the else
+def _read(self, fp, a, value, f):
+    for line in fp:
+        for prefix in a:
+            fp()
+        if (value and fp and
+            prefix > 5):
+            f()
+        else:
+            f()

test/simple_source/bug36/05_36lambda.py

@@ -0,0 +1,20 @@
+# From Python 3.6 hmac.py
+# needed to change mklambda rule
+def __init__(self, msg = None, digestmod = None):
+    self.digest_cons = lambda d='': digestmod.new(d)
+
+# From Python 3.6 functools.py
+# Bug was handling lambda for MAKE_FUNCTION_8 (closure)
+# vs to MAKE_FUNCTION_9 (pos_args + closure)
+def bug():
+    def register(cls, func=None):
+        return lambda f: register(cls, f)
+
+# From Python 3.6 configparser.py
+def items(self, d, section=5, raw=False, vars=None):
+    if vars:
+        for key, value in vars.items():
+            d[self.optionxform(key)] = value
+    d = lambda option: self._interpolation.before_get(self,
+        section, option, d[option], d)
+    return

test/simple_source/bug36/05_if_and_comp.py

@@ -0,0 +1,12 @@
+# From 3.6 base64.py
+# Bug was handling "and" condition in the presense of POP_JUMP_IF_FALSE
+# locations
+def _85encode(foldnuls, words):
+    return ['z' if foldnuls and word
+                else  'y'
+                for word in words]
+
+# From Python 3.6 enum.py
+
+def __new__(metacls, cls, bases, classdict):
+    {k: classdict[k] for k in classdict._member_names}

test/simple_source/bug36/05_while_and_if.py

@@ -0,0 +1,13 @@
+# From Python 3.6 getopt.py
+# Bug showing that "while" can have several "COME_FROMS" before loop end
+# NOTE: uncompyle6 still gets the "if"s wrong.
+def getopt(args):
+    while args and args[0] and args[0] != '-':
+        if args[0] == '--':
+            break
+        if args[0]:
+            opts = 5
+        else:
+            opts = 6
+
+    return opts

test/simple_source/bug36/10_argparse.py

@@ -0,0 +1,18 @@
+# From 3.6.4 test_argparse.py
+# Bug was in parsing ** args
+import argparse
+def test_namespace_starkwargs_notidentifier(self):
+    ns = argparse.Namespace(**{'"': 'quote'})
+    string = """Namespace(**{'"': 'quote'})"""
+    assert ns ==  string
+
+def test_namespace_kwargs_and_starkwargs_notidentifier(self):
+    ns = argparse.Namespace(a=1, **{'"': 'quote'})
+    string = """Namespace(a=1, **{'"': 'quote'})"""
+    assert ns == string
+
+
+def test_namespace(self):
+    ns = argparse.Namespace(foo=42, bar='spam')
+    string = "Namespace(bar='spam', foo=42)"
+    assert ns == string

test/simple_source/exception/04_assert_continue.py

@@ -0,0 +1,16 @@
+# Adapted from Python 3.3 idlelib/PyParse.py
+# Bug is continue flowing back to while messing up the determination
+# that it is inside an "if".
+
+# RUNNABLE!
+def _study1(i, n, ch):
+    while i == 3:
+        i = 4
+        if ch:
+            i = 10
+            assert i < 5
+            continue
+        if n:
+            return n
+
+assert _study1(3, 4, False) == 4

test/simple_source/stmts/15_assert.py

@@ -1,3 +1,5 @@
+# RUNNABLE!
+
 # Tests:
 #   2.7:
 #   assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
@@ -16,7 +18,7 @@ for method_name in ['a']:
     if method_name in ('b',):
         method = 'a'
     else:
-        assert 0, "instance installed"
+        assert True, "instance installed"
 
     methods = 'b'
 
@@ -25,5 +27,9 @@ for method_name in ['a']:
 #   if not not do_setlocal:
 #      raise AssertError
 
+# Hmmm.. this isn't strickly a bug
+
 def getpreferredencoding(do_setlocale=True):
     assert not do_setlocale
+
+getpreferredencoding(False)

test/stdlib/.gitignore

@@ -0,0 +1 @@
+/.python-version

test/stdlib/run-and-email.sh

@@ -1,13 +1,20 @@
-#!/usr/bin/bash
+#!/bin/bash
+USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-for VERSION in 2.7.14 2.6.9 ; do
+SUBJECT_PREFIX="stdlib unit testing for"
+for VERSION in 2.6.9 2.7.14 3.4.8 3.5.5 3.6.4 ; do
+    typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
-    ./runtests.sh  >$LOGFILE 2>&1
-    rc=$?
-    if ((rc == 0)); then
-	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+    if ! pyenv local $VERSION ; then
+	rc=1
     else
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+      /bin/bash ./runtests.sh  >$LOGFILE 2>&1
+      rc=$?
+    fi
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
+    else
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" $EMAIL
     fi
 done

test/stdlib/runtests.sh

@@ -48,6 +48,7 @@ case $PYVERSION in
 	;;
     2.6)
 	SKIP_TESTS=(
+	    [test_compile.py]=1  # Intermittent - sometimes works and sometimes doesn't
 	    [test_grp.py]=1      # Long test - might work Control flow?
 	    [test_opcodes.py]=1
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
@@ -77,7 +78,8 @@ case $PYVERSION in
 	    [test_curses.py]=1  # Possibly fails on its own but not detected
 	    [test_dis.py]=1   # We change line numbers - duh!
 	    [test_doctest.py]=1 # Fails on its own
-	    [test_grammar.py]=1  # Too many stmts. Handle large stmts
+	    [test_generators.py]=1  # control flow. uncompyle2 has problem here too
+	    [test_grammar.py]=1     # Too many stmts. Handle large stmts
 	    [test_io.py]=1 # Test takes too long to run
 	    [test_ioctl.py]=1 # Test takes too long to run
 	    [test_itertools.py]=1 # Fix erroneous reduction to "conditional_true".
@@ -96,9 +98,26 @@ case $PYVERSION in
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
         )
 	;;
+    3.5)
+	SKIP_TESTS=(
+	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_quopri.py]=1      # Fails in crontab environment?
+	)
+	;;
+
+    3.6)
+	SKIP_TESTS=(
+	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation
+	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_pow.py]=1         # Control flow wrt "continue"
+	    [test_quopri.py]=1      # Only fails on POWER
+	)
+	;;
     *)
-	SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1
+	SKIP_TESTS=( [test_aepack.py]=1
+		     [audiotests.py]=1
 		     [test_dis.py]=1   # We change line numbers - duh!
+		     [test_generators.py]=1  # I think string formatting of docstrings gets in the way. Not sure
 		   )
 	;;
 esac
@@ -178,7 +197,7 @@ typeset -i ALL_FILES_ENDTIME=$(date +%s)
 
 (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
 
-printf "Ran $i tests in "
+printf "Ran $i unit-test files in "
 displaytime $time_diff
 
 exit $allerrs

uncompyle6/main.py

@@ -38,7 +38,11 @@ def _get_outstream(outfile):
         os.makedirs(dir)
     except OSError:
         pass
-    return open(outfile, 'w')
+    if PYTHON_VERSION < 3.0:
+        mode = 'wb'
+    else:
+        mode = 'w'
+    return open(outfile, mode)
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
@@ -161,6 +165,7 @@ def main(in_base, out_base, files, codes, outfile=None,
 
     for filename in files:
         infile = os.path.join(in_base, filename)
+        # print("XXX", infile)
         if not os.path.exists(infile):
             sys.stderr.write("File '%s' doesn't exist. Skipped\n"
                              % infile)

uncompyle6/parser.py

@@ -431,9 +431,6 @@ class PythonParser(GenericASTBuilder):
 
         for_block ::= l_stmts_opt _come_froms JUMP_BACK
 
-        for       ::= SETUP_LOOP expr for_iter store
-                      for_block POP_BLOCK _come_froms
-
         forelsestmt ::= SETUP_LOOP expr for_iter store
                         for_block POP_BLOCK else_suite _come_froms
 
@@ -570,9 +567,6 @@ class PythonParser(GenericASTBuilder):
 
         # Positional arguments in make_function
         pos_arg ::= expr
-
-        expr32 ::= expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr
-        expr1024 ::= expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32
         '''
 
     def p_store(self, args):

uncompyle6/parsers/parse2.py

@@ -1,8 +1,21 @@
 #  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#
 #  Copyright (c) 1999 John Aycock
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-A spark grammar for Python 2.x.
+Base grammar for Python 2.x.
 
 However instead of terminal symbols being the usual ASCII text,
 e.g. 5, myvariable, "for", etc.  they are CPython Bytecode tokens,
@@ -82,6 +95,9 @@ class Python2Parser(PythonParser):
         raise_stmt2 ::= expr expr RAISE_VARARGS_2
         raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
 
+        for         ::= SETUP_LOOP expr for_iter store
+                        for_block POP_BLOCK _come_froms
+
         del_stmt ::= expr DELETE_SLICE+0
         del_stmt ::= expr expr DELETE_SLICE+1
         del_stmt ::= expr expr DELETE_SLICE+2
@@ -93,11 +109,8 @@ class Python2Parser(PythonParser):
         _mklambda ::= load_closure mklambda
         kwarg     ::= LOAD_CONST expr
 
-        kvlist ::= kvlist kv3
         kv3 ::= expr expr STORE_MAP
 
-        dict ::= BUILD_MAP kvlist
-
         classdef ::= buildclass store
 
         buildclass ::= LOAD_CONST expr mkfunc
@@ -175,6 +188,7 @@ class Python2Parser(PythonParser):
     def p_expr2(self, args):
         """
         expr ::= LOAD_LOCALS
+        expr ::= LOAD_ASSERT
         expr ::= slice0
         expr ::= slice1
         expr ::= slice2
@@ -281,19 +295,8 @@ class Python2Parser(PythonParser):
             # The order of opname listed is roughly sorted below
             if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
                 v = token.attr
-                thousands = (v//1024)
-                thirty32s = ((v//32) % 32)
-                if thirty32s > 0:
-                    rule = "expr32 ::=%s" % (' expr' * 32)
-                    self.add_unique_rule(rule, opname_base, v, customize)
-                    self.seen32 = True
-                if thousands > 0:
-                    self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),
-                                         opname_base, v, customize)
-                    self.seen1024 = True
                 collection = opname_base[opname_base.find('_')+1:].lower()
-                rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +
-                        'expr32 '*thirty32s + 'expr '*(v % 32) + opname)
+                rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname)
                 self.add_unique_rules([
                     "expr ::= %s" % collection,
                     rule], customize)
@@ -313,11 +316,9 @@ class Python2Parser(PythonParser):
                             'dict_comp_func', 0, customize)
 
                 else:
-                    kvlist_n = "kvlist_%s" % token.attr
-                    self.add_unique_rules([
-                        (kvlist_n + " ::=" + ' kv3' * token.attr),
-                        "dict ::= %s %s" % (opname, kvlist_n)
-                    ], customize)
+                    kvlist_n =  ' kv3' * token.attr
+                    rule = "dict ::= %s%s" % (opname, kvlist_n)
+                    self.addRule(rule, nop_func)
                 continue
             elif opname_base == 'BUILD_SLICE':
                 slice_num  = token.attr
@@ -503,8 +504,9 @@ class Python2Parser(PythonParser):
             self.addRule(rule, nop_func)
             pass
 
-        self.check_reduce['aug_assign1'] = 'AST'
+        self.check_reduce['raise_stmt1'] = 'tokens'
         self.check_reduce['aug_assign2'] = 'AST'
+        self.check_reduce['or'] = 'AST'
         # self.check_reduce['_stmts'] = 'AST'
 
         # Dead code testing...
@@ -520,8 +522,16 @@ class Python2Parser(PythonParser):
         # if lhs == 'while1elsestmt':
         #     from trepan.api import debug; debug()
 
-        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] == 'and':
+        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
             return True
+        elif lhs in ('raise_stmt1',):
+            # We will assme 'LOAD_ASSERT' will be handled by an assert grammar rule
+            return (tokens[first] == 'LOAD_ASSERT' and
+                    (last >= len(tokens) or tokens[last] not in
+                     ('COME_FROM', 'JUMP_BACK','JUMP_FORWARD')))
+        elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
+            expr2 = ast[2]
+            return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
         return False
 
 class Python2ParserSingle(Python2Parser, PythonParserSingle):

uncompyle6/parsers/parse25.py

@@ -65,7 +65,6 @@ class Python25Parser(Python26Parser):
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
         classdefdeco2 ::= LOAD_CONST expr mkfunc CALL_FUNCTION_0 BUILD_CLASS
         kv3 ::= expr expr STORE_MAP
-        kvlist ::= kvlist kv3
         ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
         return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
         return_if_stmt   ::= ret_expr RETURN_END_IF POP_TOP

uncompyle6/parsers/parse26.py

@@ -261,6 +261,9 @@ class Python26Parser(Python2Parser):
 
     def p_misc26(self, args):
         """
+        dict ::= BUILD_MAP kvlist
+        kvlist ::= kvlist kv3
+
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and          ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP