Merge branch 'master' into python-2.4

reinstate lots of stdlib tests which are fixed by this change.

.circleci/config.yml

@@ -1,7 +1,6 @@
 version: 2
 jobs:
   build:
-    working_directory: ~/rocky/python-uncompyle6
     parallelism: 1
     shell: /bin/bash --login
     # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
@@ -13,7 +12,7 @@ jobs:
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
     docker:
-      - image: circleci/python:3.6.9
+      - image: circleci/python:2.7
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -23,10 +22,6 @@ jobs:
     # In many cases you can simplify this from what is generated here.
     # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
     - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    # This is based on your 1.0 configuration file or project settings
-    - run:
-        working_directory: ~/rocky/python-uncompyle6
-        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
@@ -37,9 +32,8 @@ jobs:
           - v2-dependencies-
 
     # This is based on your 1.0 configuration file or project settings
-    - run: pip install --user --upgrade setuptools
-    - run: pip install --user -e .
-    - run: pip install --user -r requirements-dev.txt
+    - run:
+        command: sudo easy_install xdis spark-parser && sudo pip install -e . && sudo pip install -r requirements-dev.txt
 
     # Save dependency cache
     - save_cache:
@@ -57,8 +51,8 @@ jobs:
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
-    - run: sudo python ./setup.py develop && make check-3.6
-    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
+    - run: sudo python ./setup.py develop && make check-2.7
+    - run: cd test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
     # Save test results

.travis.yml

@@ -1,16 +1,7 @@
 language: python
 
 python:
-  - '3.5'
-  - '2.7'
-  - '3.4'
-  - '3.6'
-  - '3.8'
-
-matrix:
-  include:
-    - python: '3.7'
-      dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
+  - 2.7 # this is a cheat here because travis doesn't do 2.4-2.6
 
 install:
 - pip install -e .

Makefile

@@ -40,8 +40,9 @@ check-3.0 check-3.1 check-3.2 check-3.6:
 check-3.7: pytest
 	$(MAKE) -C test check
 
-check-3.8:
-	$(MAKE) -C test check
+#:Tests for Python 2.4-2.5 (don't have pytest)
+check-2.4 check-2.5:
+	$(MAKE) -C test $@
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now

NEWS.md

@@ -1,17 +1,3 @@
-3.6.5: 2020-4-1 April Fool
-==========================
-
-Back port some of the changes in decompile3 here which mostly helps 3.7 and 3.8 decompilation, although this may also help 3.6ish versions too.
-
-- Handle nested `async for in for...`  and Better async comprehension detection via `xdis`.  Still more work is needed.
-- include token number in listings when `-g` and there is a parser error
-- remove unneded `Makefile`s now that remake 4.3+1.5dbg is a thing that has `-c`
-- Bug in finding annotations in functions with docstrings
-- Fix bug found by 2.4 sre_parse.py testing
-- Fix `transform` module's  `ifelseif` bugs
-- Fix bug in 3.0 name module detection
-- Fix docstring detection
-
 3.6.4: 2020-2-9 Plateau
 =======================
 

README.rst

@@ -54,7 +54,7 @@ only; another patched that and handled only 3.3.  You get the
 idea. This code pulls all of these forks together and *moves
 forward*. There is some serious refactoring and cleanup in this code
 base over those old forks. Even more experimental refactoring is going
-on in decompyle3_.
+on in decompile3_.
 
 This demonstrably does the best in decompiling Python across all
 Python versions. And even when there is another project that only
@@ -89,8 +89,7 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    $ pip install -e .  # set up to run from source tree
-                        # Or if you want to install instead
+    $ pip install -e .  # set up to run from source tree, or...
     $ python setup.py install # may need sudo
 
 A GNU makefile is also provided so :code:`make install` (possibly as root or
@@ -101,7 +100,7 @@ Running Tests
 
 ::
 
-   make check
+   $  make check
 
 A GNU makefile has been added to smooth over setting running the right
 command, and running tests from fastest to slowest.
@@ -252,14 +251,11 @@ See Also
 .. _uncompyle2: https://github.com/wibiti/uncompyle2
 .. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
-.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
-		 :target: https://travis-ci.org/rocky/python-uncompyle6
-.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
-		 :target: https://repology.org/project/python:uncompyle6/versions
+.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
-.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
-		 :target: https://badge.fury.io/py/uncompyle6
+.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg :target: https://badge.fury.io/py/uncompyle6
 .. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -58,7 +58,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.2.4, < 4.3.0"]
+                      "xdis >= 4.2.2, < 4.3.0"]
 
 license            = "GPL3"
 mailing_list       = "python-debugger@googlegroups.com"

admin-tools/check-older-versions.sh

@@ -20,8 +20,9 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && python setup.py develop
-    if ! make check ; then
+    if ! make check-short ; then
 	exit $?
     fi
     echo === $version ===
 done
+make check

admin-tools/how-to-make-a-release.md

@@ -55,10 +55,9 @@
 # Make packages and tag
 
     $ . ./admin-tools/make-dist-older.sh
-	$ pyenv local 3.8.2
+	$ pyenv local 3.8.1
 	$ twine check dist/uncompyle6-$VERSION*
     $ git tag release-python-2.4-$VERSION
-	$ twine check dist/uncompyle6-$VERSION*
     $ . ./admin-tools/make-dist-newer.sh
 	$ twine check dist/uncompyle6-$VERSION*
 

admin-tools/how-to-make-a-release.txt

@@ -0,0 +1,46 @@
+git pull
+
+Change version in uncompyle6/version.py
+source uncompyle6/version.py
+echo $VERSION
+git commit -m"Get ready for release $VERSION" .
+
+Update ChangeLog:
+  make ChangeLog
+
+Update NEWS from ChangeLog
+make check
+
+git commit --amend .
+
+git push
+
+Make sure pyenv is running
+# Pyenv
+
+source admin-tools/check-newer-versions.sh
+
+
+# Switch to python-2.4 and build that first...
+source admin-tools/setup-python-2.4
+
+rm ChangeLog
+git merge master
+
+Update NEWS from master branch
+
+git commit -m"Get ready for release $VERSION" .
+
+source admin-tools/check-older-versions.sh
+source admin-tools/check-newer-versions.sh
+
+make-dist-older.sh
+
+git tag release-python-2.4-$VERSION
+
+./make-dist-newer.sh
+
+git tag release-$VERSION
+
+
+twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.9 3.6.10 2.6.9 3.3.7 2.7.17 3.2.6 3.1.5 3.4.10 3.7.7 3.8.2'
+export PYVERSIONS='3.5.9 3.6.10 2.6.9 3.3.7 2.7.17 3.2.6 3.1.5 3.4.10 3.7.6 3.8.1'

admin-tools/pyenv-older-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6 2.6.9'
+export PYVERSIONS='2.4.6 2.5.6'

admin-tools/setup-master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-PYTHON_VERSION=3.7.7
+PYTHON_VERSION=3.7.6
 
 # FIXME put some of the below in a common routine
 function finish {

admin-tools/setup-python-2.4.sh

@@ -15,3 +15,4 @@ cd $fulldir/..
     git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
 cd $owd
 rm -v */.python-version || true
+pyenv local $PYTHON_VERSION

admin-tools/update-sources.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd $(dirname ${BASH_SOURCE[0]})/..
+git pull

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.__code__ if PYTHON3 else fn.func_code
+    code = fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_fjt.py

@@ -20,7 +20,7 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.__code__
+    code = bug.func_code
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         scan.build_instructions(code)

pytest/test_grammar.py

@@ -45,6 +45,9 @@ def test_grammar():
         expect_lhs.add("kvlist")
         expect_lhs.add("kv3")
         unused_rhs.add("dict")
+    else:
+        # NOTE: this may disappear
+        expect_lhs.add("except_handler_else")
 
         if PYTHON_VERSION < 3.7 and PYTHON_VERSION != 2.7:
             # NOTE: this may disappear

pytest/test_pysource.py-notyet

@@ -8,12 +8,8 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
-    def iteritems(d):
-        return d.items()
 else:
     from StringIO import StringIO
-    def iteritems(d):
-        return d.iteritems()
 
 from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
@@ -30,7 +26,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_DIRECT, TABLE_R,
+    TABLE_R, TABLE_DIRECT,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -44,7 +40,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in iteritems(t):
+        for k, entry in t.iteritems():
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,7 +1,4 @@
-import pytest
 from uncompyle6 import PYTHON_VERSION, code_deparse
-pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
-                              reason="need at least Python 2.7")
 
 if PYTHON_VERSION > 2.6:
     def test_single_mode():

pytest/test_token.py

@@ -5,7 +5,7 @@ from uncompyle6.scanners.tok import Token
 def test_token():
     # Test token formatting of: LOAD_CONST None
     t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
-    expect = "                0  LOAD_CONST               None"
+    expect = "             0  LOAD_CONST               None"
     # print(t.format())
     assert t
     assert t.format() == expect
@@ -17,7 +17,7 @@ def test_token():
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
     t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
-    expect = "                1  LOAD_CONST               False"
+    expect = "             1  LOAD_CONST               False"
     assert t.format() == expect
 
 

pytest/testdata/if-2.7.right

@@ -1,12 +1,12 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_if.py
 
- L.   6         0  LOAD_NAME             0  'True'
-                3  POP_JUMP_IF_FALSE    15  'to 15'
+   6         0  LOAD_NAME             0  'True'
+             3  POP_JUMP_IF_FALSE    15  'to 15'
 
- L.   7         6  LOAD_NAME             1  'False'
-                9  STORE_NAME            2  'b'
-               12  JUMP_FORWARD          0  'to 15'
-             15_0  COME_FROM            12  '12'
-               15  LOAD_CONST               None
-               18  RETURN_VALUE     
+   7         6  LOAD_NAME             1  'False'
+             9  STORE_NAME            2  'b'
+            12  JUMP_FORWARD          0  'to 15'
+          15_0  COME_FROM            12  '12'
+            15  LOAD_CONST               None
+            18  RETURN_VALUE

pytest/testdata/ifelse-2.7.right

@@ -1,15 +1,15 @@
 # Python 2.7
 # Embedded file name: simple_source/branching/05_ifelse.py
 
- L.   3         0  LOAD_NAME             0  'True'
-                3  POP_JUMP_IF_FALSE    15  'to 15'
+   3         0  LOAD_NAME             0  'True'
+             3  POP_JUMP_IF_FALSE    15  'to 15'
 
- L.   4         6  LOAD_CONST               1
-                9  STORE_NAME            1  'b'
-               12  JUMP_FORWARD          6  'to 21'
+   4         6  LOAD_CONST               1
+             9  STORE_NAME            1  'b'
+            12  JUMP_FORWARD          6  'to 21'
 
- L.   6        15  LOAD_CONST               2
-               18  STORE_NAME            2  'd'
-             21_0  COME_FROM            12  '12'
-               21  LOAD_CONST               None
-               24  RETURN_VALUE     
+   6        15  LOAD_CONST               2
+            18  STORE_NAME            2  'd'
+          21_0  COME_FROM            12  '12'
+            21  LOAD_CONST               None
+            24  RETURN_VALUE

setup.py

@@ -1,16 +1,16 @@
 #!/usr/bin/env python
-import sys
-
 """Setup script for the 'uncompyle6' distribution."""
 
+import sys
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 9)):
-    mess = "Python Release 2.6 .. 3.9 are supported in this code branch."
-    if ((2, 4) <= SYS_VERSION <= (2, 7)):
-        mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
+if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
+    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
+    if ((3, 2) <= SYS_VERSION  <= (3, 8)):
+        mess += ("\nFor your Python, version %s, use the master code/branch." %
                  sys.version[0:3])
-    elif SYS_VERSION < (2, 4):
-        mess += ("\nThis package is not supported for Python version %s."
+    else:
+        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
                  % sys.version[0:3])
     print(mess)
     raise Exception(mess)

test-unit/test_grammar.py

@@ -0,0 +1,55 @@
+import re
+import unittest
+from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
+from uncompyle6.parser import get_python_parser, python_parser
+
+class TestGrammar(unittest.TestCase):
+    def test_grammar(self):
+
+        def check_tokens(tokens, opcode_set):
+            remain_tokens = set(tokens) - opcode_set
+            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+            remain_tokens = set(remain_tokens) - opcode_set
+            self.assertEqual(remain_tokens, set([]),
+                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
+
+        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
+        (lhs, rhs, tokens,
+         right_recursive, dup_rhs) = p.check_sets()
+        expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
+        unused_rhs = set(['list', 'call', 'mkfunc',
+                          'mklambda',
+                          'unpack',])
+
+        expect_right_recursive = frozenset([('designList',
+                                             ('store', 'DUP_TOP', 'designList'))])
+        expect_lhs.add('kwarg')
+
+        self.assertEqual(expect_lhs, set(lhs))
+        self.assertEqual(unused_rhs, set(rhs))
+        self.assertEqual(expect_right_recursive, right_recursive)
+
+        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+                                    ('LOAD_CONST',),
+                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
+
+        reduced_dup_rhs = {}
+        for k in dup_rhs:
+            if k not in expect_dup_rhs:
+                reduced_dup_rhs[k] = dup_rhs[k]
+                pass
+            pass
+        for k in reduced_dup_rhs:
+            print(k, reduced_dup_rhs[k])
+        # assert not reduced_dup_rhs, reduced_dup_rhs
+
+    def test_dup_rule(self):
+        import inspect
+        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
+                      is_pypy=IS_PYPY,
+                      parser_debug={
+                          'dups': True, 'transition': False, 'reduce': False,
+                          'rules': False, 'errorstack': None, 'context': True})
+if __name__ == '__main__':
+    unittest.main()

test/Makefile

@@ -30,7 +30,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
+check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -72,13 +72,13 @@ check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
 
-#: Run working tests from Python 3.8
-check-3.8: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
+# #: Run working tests from Python 3.8
+# check-3.8: check-bytecode
+# 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+# 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
 
 # FIXME
-#: this is called when running under pypy3.5-5.8.0, pypy2-5.6.0, or pypy3.6-7.3.0
+#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
 5.8 5.6:
 
 #: Check deparsing only, but from a different Python version
@@ -99,8 +99,8 @@ check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7  \
-	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
+        --bytecode-3.7 \
+	--bytecode-pypy3.2 --bytecode-pypy3.6
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -166,6 +166,7 @@ check-bytecode-2.3:
 
 #: Check deparsing Python 2.4
 check-bytecode-2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-2.4-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
@@ -294,16 +295,20 @@ check-bytecode-3.7:
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify
 
-#: Check deparsing Python 3.8
-check-bytecode-3.8:
-	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
+# #: Check deparsing Python 3.8
+# check-bytecode-3.8:
+# 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+# 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --syntax-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
+#: Run longer Python 2.6's lib files known to be okay
+check-2.4-ok:
+	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
+
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --syntax-verify $(COMPILE)
@@ -339,8 +344,8 @@ check-bytecode-pypy3.6: 7.1
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 
 #: PyPy 5.0.x with Python 3.6.9
-check-bytecode-pypy3.6: 7.2 7.3
-7.3 7.2:
+check-bytecode-pypy3.6: 7.2
+7.2:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 

test/dis-compare.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
 
 
 import dis, os.path

test/run-and-email.sh

@@ -3,55 +3,32 @@
 function displaytime {
     printf "ran in "
     local T=$1
-    ((D=T/60/60/24))
-    ((H=T/60/60%24))
-    ((M=T/60%60))
-    ((S=T%60))
-    (( D > 0 )) && printf '%d days ' $D
-    (( H > 0 )) && printf '%d hours ' $H
-    (( M > 0 )) && printf '%d minutes ' $M
-    (( D > 0 || H > 0 || M > 0 )) && printf 'and '
+    local D=$((T/60/60/24))
+    local H=$((T/60/60%24))
+    local M=$((T/60%60))
+    local S=$((T%60))
+    (( $D > 0 )) && printf '%d days ' $D
+    (( $H > 0 )) && printf '%d hours ' $H
+    (( $M > 0 )) && printf '%d minutes ' $M
+    (( $D > 0 || $H > 0 || $M > 0 )) && printf 'and '
     printf '%d seconds\n' $S
 }
 
-bs=${BASH_SOURCE[0]}
-if [[ $0 != $bs ]] ; then
-    echo "This script should not be *sourced* but run through bash"
-    exit 1
-fi
-
-mydir=$(dirname $bs)
-cd $mydir
-
-branch=$(cat ../.git/HEAD | cut -d'/' -f 3)
-if [[ $branch == 'python-2.4' ]]; then
-    . ../admin-tools/pyenv-older-versions
-elif [[ $branch == 'master' ]]; then
-    . ../admin-tools/pyenv-newer-versions
-else
-    echo &1>2 "Error git branch should either be 'master' or 'python-2.4'; got: '$branch'"
-    exit 1
-fi
-
-MAIN="test_pyenvlib.py"
+. ../admin-tools/pyenv-newer-versions
 
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-WHAT="uncompyle6 ${MAIN}"
 MAX_TESTS=${MAX_TESTS:-800}
-export BATCH=1
-
 typeset -i RUN_STARTTIME=$(date +%s)
 
 # PYVERSIONS="3.5.6"
-MAILBODY=/tmp/${MAIN}-mailbody-$$.txt
-# for VERSION in 3.3.7 ; do
+actual_versions=""
 for VERSION in $PYVERSIONS ; do
     typeset -i rc=0
-    LOGFILE=/tmp/${MAIN}-$VERSION-$$.log
+    LOGFILE=/tmp/pyenvlib-$VERSION-$$.log
 
     case "$VERSION" in
-	3.7.7 | 3.8.2 | 3.1.5 | 3.0.1 )
+	3.7.6 | 3.8.1 | 3.1.5 | 3.0.1 )
 	    continue
 	    ;;
 	3.5.9 )
@@ -69,12 +46,6 @@ for VERSION in $PYVERSIONS ; do
 	3.6.10 )
 	    MAX_TESTS=1300  # about 2139 exist
 	    ;;
-	2.4.6 )
-	    MAX_TESTS=600
-	    ;;
-	2.5.6 )
-	    MAX_TESTS=600
-	    ;;
 	2.6.9 )
 	    MAX_TESTS=1300
 	    ;;
@@ -87,13 +58,11 @@ for VERSION in $PYVERSIONS ; do
 
     if ! pyenv local $VERSION ; then
 	rc=1
-	mailbody_line="pyenv local $VERSION not installed"
-	echo $mailbody_line >> $MAILBODY
     else
       echo Python Version $(pyenv local) > $LOGFILE
       echo "" >> $LOGFILE
       typeset -i ALL_FILES_STARTTIME=$(date +%s)
-      cmd="python ./${MAIN} --max ${MAX_TESTS} --syntax-verify --$VERSION"
+      cmd="python ./test_pyenvlib.py --max ${MAX_TESTS} --syntax-verify --$VERSION"
       echo "$cmd" >>$LOGFILE 2>&1
       $cmd >>$LOGFILE 2>&1
       rc=$?
@@ -103,25 +72,20 @@ for VERSION in $PYVERSIONS ; do
 
       typeset -i ALL_FILES_ENDTIME=$(date +%s)
       (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
-      time_str=$(displaytime $time_diff)
-      echo ${time_str}. >> $LOGFILE
+      displaytime $time_diff >> $LOGFILE
     fi
 
-    SUBJECT_PREFIX="$WHAT (max $MAX_TESTS) for"
+    SUBJECT_PREFIX="pyenv weak verify (max $MAX_TESTS) for"
     if ((rc == 0)); then
-	mailbody_line="Python $VERSION ok; ${time_str}."
 	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
     else
-	mailbody_line="Python $VERSION failed; ${time_str}."
 	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
-	tail -v $LOGFILE | mail -s "$HOST $SUBJECT_PREFIX $VERSION not ok" ${EMAIL}
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${EMAIL}
     fi
-    echo $mailbody_line >> $MAILBODY
     rm .python-version
 done
 
 typeset -i RUN_ENDTIME=$(date +%s)
 (( time_diff =  RUN_ENDTIME - RUN_STARTTIME))
 elapsed_time=$(displaytime $time_diff)
-echo "${WHAT} complete; ${elapsed_time}." >> $MAILBODY
-cat $MAILBODY | mail -s "$HOST $WHAT ${elapsed_time}." ${EMAIL}
+echo "Run complete $elapsed_time for versions $actual_versions" | mail -s "pyenv weak verify in $elapsed_time" ${EMAIL}

test/simple-uncompyle-code-test.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
-
+from uncompyle6 import uncompyle
 from uncompyle6.main import decompile
 from xdis.magics import sysinfo2float
 import sys, inspect

test/simple_source/bug25/02_yield_bug.py

@@ -0,0 +1,7 @@
+# From 2.4 test_array.py
+# In Python 2.4 and earlier "yield" is not valid and instead
+# we must use "yield None". Bug was not adding "None"
+
+def yield_bug():
+    yield None
+    return

test/simple_source/bug25/04_try_else_confuse.py

@@ -0,0 +1,19 @@
+# From 2.4 test_sax.py
+# Bug was distinguishing try from try/else
+
+def verify_empty_attrs():
+    gvqk = 3
+    try:
+        gvk = 1/0
+    except ZeroDivisionError:
+        gvk = 1
+
+    try:
+        gvqk = 0
+    except KeyError:
+        gvqk = 1
+
+    # If try/else was used above the return will be 4
+    return gvk + gvqk
+
+assert 1 == verify_empty_attrs()

test/simple_source/bug31/04_def_annotate.py

@@ -66,14 +66,6 @@ def div(a: dict(type=float, help='the dividend'),
     """Divide a by b"""
     return a / b
 
-# From 3.7.6 functools.py
-# Bug is in picking up the annotation.
-def f(a:"This is a new annotation"):
-    """This is a test"""
-    assert f.__annotations__['a'] == "This is a new annotation"
-
-f(5)
-
 class TestSignatureObject1():
     def test_signature_on_wkwonly(self):
         def test(*, a:float, b:str, c:str = 'test', **kwargs: int) -> int:

test/simple_source/bug37/02_async_for_generator.py

@@ -6,11 +6,3 @@ def make_arange(n):
 
 async def run(m):
     return [i async for i in m]
-
-# From 3.7.6 test_coroutines.py
-async def run_list(pair, f):
-    return [i for pair in p async for i in f]
-
-# FIXME: add this. It works in decompyle3
-# async def run_gen():
-#     return (i async for i in f if 0 < i < 4)

test/simple_source/expression/09_yield_from.py

@@ -17,9 +17,9 @@ def ybug(g):
 # From 3.5.1 _wakrefset.py
 #
 #  3.5:
-#  with ::= expr SETUP_WITH POP_TOP suite_stmts_opt
-#           POP_BLOCK LOAD_CONST COME_FROM
-#           WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+#  withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
+#              POP_BLOCK LOAD_CONST COME_FROM
+#              WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
 
 def __iter__(self, IterationGuard):

test/simple_source/looping/08_while1_if_continue.py

@@ -7,40 +7,3 @@ def readline (self):
                 continue
 
         return
-
-# From 2.4.6 sre.py
-# Bug has to do with "break" not being recognized
-# and is a JUMP_FORWARD.
-def _parse(a, b, source, state):
-    while 1:
-        if b:
-            while 1:
-                break
-        else:
-            raise
-
-def _parse2(source, state):
-    while 1:
-        if a:
-            if b:
-                while 1:
-                    this = 1
-                    break
-                continue
-
-        while 1:
-            if b:
-                break
-
-        x = 3
-
-# Bug was in 2.3 decompilation
-def _parse3(source, state):
-    while 1:
-        if a:
-            if b:
-                x = 1
-                while 1:
-                    if a:
-                        break
-                    raise

test/simple_source/stmts/03_if_elif.py

@@ -1,13 +1,11 @@
 # 2.6.9 symbols.py
 # Bug in 2.6 is having multple COME_FROMs due to the
 # "and" in the "if" clause
-
-# RUNNABLE
 if __name__:
     if __file__ and __name__:
         pass
-    elif not __name__:
-        assert False
+    elif __name__:
+        pass
 
 # 2.6.9 transformer.py
 # Bug in 2.6 is multple COME_FROMs as a result
@@ -23,20 +21,3 @@ elif __file__:
     assert __name__ or __file__
 else:
     pass
-
-# From 3.3.7 test_binop.py
-# Bug was in ifelsestmt(c) ensuring b+=5 is not in "else"
-# Also note: ifelsetmtc should not have been used since this
-# this is not in a loop!
-def __floordiv__(a, b):
-    if a:
-        b += 1
-    elif not b:
-        return a
-    b += 5
-    return b
-
-assert __floordiv__(1, 1) == 7
-assert __floordiv__(1, 0) == 6
-assert __floordiv__(0, 3) == 8
-assert __floordiv__(0, 0) == 0

test/stdlib/2.5-exclude.sh

@@ -81,6 +81,6 @@ SKIP_TESTS=(
 )
 # About 265 tests in 14 minutes
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     SKIP_TESTS[test_doctest.py]=1 # Fails on ppc64le
 fi

test/stdlib/2.6-exclude.sh

@@ -85,7 +85,7 @@ SKIP_TESTS=(
 )
 # About 305 unit-test files in about 12 minutes
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     # Fails in crontab environment?
     # Figure out what's up here
     SKIP_TESTS[test_aifc.py]=1

test/stdlib/2.7-exclude.sh

@@ -1,6 +1,4 @@
 SKIP_TESTS=(
-    [test_cgi.py]=1 # FIXME: Works on c90ff51
-
     [test_bsddb3.py]=1 # test takes too long to run: 110 seconds
     [test_compile.py]=1  # Code introspects on co_consts in a non-decompilable way
     [test_curses.py]=1  # Possibly fails on its own but not detected
@@ -27,7 +25,6 @@ SKIP_TESTS=(
     [test_ssl.py]=1  #
     [test_subprocess.py]=1 # Runs ok but takes 22 seconds
     [test_sys_settrace.py]=1 # Line numbers are expected to be different
-
     [test_tokenize.py]=1 # test takes too long to run: 19 seconds
     [test_traceback.py]=1 # Line numbers change - duh.
     [test_unicode.py]=1  # Too long to run 11 seconds
@@ -37,7 +34,7 @@ SKIP_TESTS=(
 )
 # 334 unit-test files in about 15 minutes
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     # Fails in crontab environment?
     # Figure out what's up here
     SKIP_TESTS[test_array.py]=1
@@ -46,7 +43,6 @@ if (( BATCH )) ; then
     SKIP_TESTS[test_doctest2.py]=1 # a POWER thing?
     SKIP_TESTS[test_httplib.py]=1  # Ok, but POWER has problems with this
     SKIP_TESTS[test_pdb.py]=1 # Ok, but POWER has problems with this
-    SKIP_TESTS[test_tarfile.py]=1 # test can take over 15 seconds to run on an overloaded POWER7 system
 
     # SyntaxError: Non-ASCII character '\xdd' in file test_base64.py on line 153, but no encoding declared; see http://www.python.org/peps/pep-0263.html for details
     SKIP_TESTS[test_base64.py]=1

test/stdlib/3.2-exclude.sh

@@ -31,7 +31,7 @@ SKIP_TESTS=(
 
 )
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     # Fails in crontab environment?
     # Figure out what's up here
     SKIP_TESTS[test_exception_variations.py]=1

test/stdlib/3.3-exclude.sh

@@ -1,9 +1,8 @@
 SKIP_TESTS=(
-    [test_buffer.py]=1  # FIXME: Works on c90ff51
-    [test_cmath.py]=1  # FIXME: Works on c90ff51
-
     [test_atexit.py]=1  # The atexit test starting at 3.3 looks for specific comments in error lines
 
+    [test_buffer.py]=1  # parse error
+
     [test_cmd_line.py]=1 # too long?
     [test_concurrent_futures.py]=1  # too long?
 
@@ -31,7 +30,6 @@ SKIP_TESTS=(
 
     [test_nntplib.py]=1
 
-    [test_pep352.py]=1  # test failures
     [test_peepholer.py]=1
     [test_poll.py]=1  # test takes too long to run: 11 seconds
     [test_pty.py]=1  # FIXME: Needs grammar loop isolation separation
@@ -59,7 +57,7 @@ SKIP_TESTS=(
 )
 # About 300 unit-test files in about 20 minutes
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     SKIP_TESTS[test_ftplib.py]=1  # Runs too long on POWER; over 15 seconds
     SKIP_TESTS[test_idle.py]=1  # No tk
     SKIP_TESTS[test_pep352.py]=1  # UnicodeDecodeError may be funny on weird environments

test/stdlib/3.4-exclude.sh

@@ -1,18 +1,11 @@
 SKIP_TESTS=(
-    [test_buffer.py]=1  # FIXME: Works on c90ff51
-    [test_cmath.py]=1  # FIXME: Works on c90ff51
-    [test_strftime.py]=1  # FIXME: Works on c90ff51
-
     [test___all__.py]=1  # it fails on its own
     [test_atexit.py]=1  # The atexit test looks for specific comments in error lines
-
     [test_cmd_line.py]=1 # takes too long to run
     [test_concurrent_futures.py]=1  # too long?
-
     [test_configparser.py]=1  # Doesn't terminate
     [test_ctypes.py]=1 # it fails on its own
     [test_curses.py]=1 # Investigate
-
     [test_dbm_gnu.py]=1   # fails on its own
     [test_devpoll.py]=1 # it fails on its own
     [test_descr.py]=1   # test assertion errors
@@ -21,44 +14,33 @@ SKIP_TESTS=(
     [test_doctest2.py]=1
     [test_doctest.py]=1  # test assert failures
     [test_docxmlrpc.py]=1
-
     [test_enum.py]=1 # compile syntax?
     [test_exceptions.py]=1
-
     [test_faulthandler.py]=1
     [test_file_eintr.py]=1   # parse error
     [test_fork1.py]=1 # too long
-
     [test_gdb.py]=1 # it fails on its own
     [test_grammar.py]=1 # parse error
-
     [test_httplib.py]=1 # it fails on its own
-
     [test_import.py]=1 # it fails on its own
     [test_io.py]=1
     [test_ioctl.py]=1 # it fails on its own
     [test_inspect.py]=1 # Syntax error Investigate
-
     [test_logging.py]=1 # Too long to run
     [test_long.py]=1 # FIXME: Works on c90ff51
-
     [test_modulefinder.py]=1  # test assertion error
     [test_multiprocessing_fork.py]=1 # doesn't terminate
     [test_multiprocessing_forkserver.py]=1 # doesn't terminate
     [test_multiprocessing_main_handling.py]=1  # doesn't terminate
     [test_multiprocessing_spawn.py]=1 # doesn't terminate
-
     [test_nntplib.py]=1 # too long to run
-
     [test_peepholer.py]=1 # control flow?
     [test_pep352.py]=1 # test assert failures
     [test_pickle.py]=1 # test assert failures
     [test_pkgimport.py]=1 # long
     [test_poll.py]=1 # Too long to run: 11 seconds
     [test_pydoc.py]=1 # test assertion failures
-
     [test_runpy.py]=1 # Too long:
-
     [test_select.py]=1 # Too long: 11 seconds
     [test_selectors.py]=1  # Too long: 11 seconds
     [test_signal.py]=1 # Too long: 22 seconds
@@ -68,15 +50,12 @@ SKIP_TESTS=(
     [test_subprocess.py]=1 # Too long
     [test_symtable.py]=1 # Investigate bad output
     [test_sys_settrace.py]=1 # test assert failures
-
     [test_tcl.py]=1 # May be implementation specific. On POWER though it fails
     [test_threading.py]=1 # Too long
     [test_threadsignals.py]=1 # Too long to run: 12 seconds
     [test_timeout.py]=1 # Too long to run: 19 seconds
     [test_traceback.py]=1 # introspects on code
-
     [test_urllib2net.py]=1 # Doesn't terminate
-
     [test_zipfile64.py]=1
     [test_zlib.py]=1
 )

test/stdlib/3.5-exclude.sh

@@ -1,7 +1,11 @@
 SKIP_TESTS=(
     [test_buffer.py]=1  # FIXME: Works on c90ff51
-    [test_platform.py]=1  # FIXME: Works on c90ff51
-    [test_pyclbr.py]=1  # FIXME: Works on c90ff51
+    [test_decorators.py]=1  # FIXME: Works on c90ff51
+    [test_platform.py]=1 # FIXME: Works on c90ff51
+    [test_pyclbr.py]=1 # FIXME: Works on c90ff51
+    [test_tempfile.py]=1 # FIXME: Works on c90ff51
+    [test_uu.py]=1 # FIXME: Works on c90ff51
+    [test_ftplib.py]=1 # FIXME: Works on c90ff51
 
     [test___all__.py]=1 # it fails on its own
     [test_aifc.py]=1  #
@@ -96,7 +100,6 @@ SKIP_TESTS=(
     [test_sys_settrace.py]=1 # test assert fail
 
     [test_tcl.py]=1  # it fails on its own
-    [test_tempfile.py]=1  # test assertion failures
     [test_thread.py]=1
     [test_threading.py]=1
     [test_timeout.py]=1
@@ -112,7 +115,6 @@ SKIP_TESTS=(
     [test_urllib2net.py]=1 # it fails on its own
     [test_urllibnet.py]=1 # it fails on its own
     [test_urlparse.py]=1 # test assert error
-    [test_uu.py]=1 # test assert error
 
     [test_winreg.py]=1 # it fails on its own
     [test_winsound.py]=1 # it fails on its own
@@ -127,7 +129,7 @@ SKIP_TESTS=(
 )
 # About 260 unit-test in about 16 minutes
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     SKIP_TESTS[test_asyncore.py]=1 # Ok, but takes more than 15 seconds to run
     SKIP_TESTS[test_bisect.py]=1
     SKIP_TESTS[test_compileall.py]=1  # Something weird on POWER
@@ -135,11 +137,10 @@ if (( BATCH )) ; then
     SKIP_TESTS[test_distutils.py]=1
 
     SKIP_TESTS[test_exception_variations.py]=1
-    SKIP_TESTS[test_ioctl.py]=1 # it fails on its own
     SKIP_TESTS[test_poplib.py]=1 # May be a result of POWER installation
 
     SKIP_TESTS[test_quopri.py]=1
-    SKIP_TESTS[test_sysconfig.py]=1 # POWER extension fails
+    SKIP_TESTS[test_ioctl.py]=1 # it fails on its own
     SKIP_TESTS[test_tarfile.py]=1 # too long to run on POWER 15 secs
     SKIP_TESTS[test_venv.py]=1 # takes too long 11 seconds
 fi

test/stdlib/3.6-exclude.sh

@@ -1,12 +1,15 @@
 SKIP_TESTS=(
     [test_ast.py]=1 # FIXME: Works on c90ff51
-    [test_cmath.py]=1  # FIXME: Works on c90ff51
+    [test_binop.py]=1 # FIXME: Works on c90ff51
+    [test_complex.py]=1 # FIXME: Works on c90ff51
+    [test_decorators.py]=1 # FIXME: Works on c90ff51
     [test_format.py]=1 # FIXME: Works on c90ff51
     [test_ftplib.py]=1 # FIXME: Works on c90ff51
     [test_slice.py]=1 # FIXME: Works on c90ff51
     [test_sort.py]=1 # FIXME: Works on c90ff51
+    [test_strftime.py]=1 # FIXME: Works on c90ff51
     [test_timeit.py]=1 # FIXME: Works on c90ff51
-    [test_os.py]=1 # parse error FIXME: Works on c90ff51
+    [test_os.py]=1 # FIXME: Works on c90ff51
 
     [test___all__.py]=1  # it fails on its own
     [test_aifc.py]=1  #
@@ -48,7 +51,6 @@ SKIP_TESTS=(
     [test_datetime.py]=1 # it fails on its own
     [test_dbm_ndbm.py]=1 # it fails on its own
     [test_decimal.py]=1
-    [test_decorators.py]=1 # control-flow failures
     [test_descr.py]=1   # syntax error: Investigate
     [test_devpoll.py]=1 # it fails on its own
     [test_dict.py]=1 # it fails on its own
@@ -106,8 +108,7 @@ SKIP_TESTS=(
     [test_nntplib.py]=1 # test takes too long to run: 31 seconds
     [test_normalization.py]=1 # it fails on its own
 
-    [test_optparse.py]=1 # test fails
-    [test_ordered_dict.py]=1 # it fails on its own
+    [test_ordered_dict.py]= # it fails on its own
     [test_ossaudiodev.py]=1 # it fails on its own
 
     [test_pdb.py]=1 # Probably introspection
@@ -148,7 +149,6 @@ SKIP_TESTS=(
     [test_startfile.py]=1 # it fails on its own
     [test_statistics.py]=1 # it fails on its own
     [test_string_literals.py]=1
-    [test_strftime.py]=1 # test assertion failures
     [test_strtod.py]=1 # it fails on its own
     [test_struct.py]=1  # test assertion errors
     [test_subprocess.py]=1
@@ -202,10 +202,8 @@ SKIP_TESTS=(
 )
 # 236 unit-test files in about 13 minutes
 
-if (( BATCH )) ; then
+if (( batch )) ; then
     SKIP_TESTS[test_codeccallbacks.py]=1
-    SKIP_TESTS[test_complex.py]=1 # Something funky with POWER8
-
     # locale on test machine is probably customized
     SKIP_TESTS[test__locale.py]=1
 fi