Administrivia

correct "return None" semantic action

.circleci/config.yml

@@ -1,6 +1,7 @@
 version: 2
 jobs:
   build:
+    working_directory: ~/rocky/python-uncompyle6
     parallelism: 1
     shell: /bin/bash --login
     # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
@@ -12,7 +13,7 @@ jobs:
     # To see the list of pre-built images that CircleCI provides for most common languages see
     # https://circleci.com/docs/2.0/circleci-images/
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.6.9
     steps:
     # Machine Setup
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
@@ -22,6 +23,10 @@ jobs:
     # In many cases you can simplify this from what is generated here.
     # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
     - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
+    # This is based on your 1.0 configuration file or project settings
+    - run:
+        working_directory: ~/rocky/python-uncompyle6
+        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
     # Dependencies
     #   This would typically go in either a build or a build-and-test job when using workflows
     # Restore the dependency cache
@@ -31,9 +36,9 @@ jobs:
           # fallback to using the latest cache if no exact match is found
           - v2-dependencies-
 
-    # This is based on your 1.0 configuration file or project settings
-    - run:
-        command: sudo easy_install xdis spark-parser && sudo pip install -e . && sudo pip install -r requirements-dev.txt
+    - run: pip install --user --upgrade setuptools
+    - run: pip install --user -e .
+    - run: pip install --user -r requirements-dev.txt
 
     # Save dependency cache
     - save_cache:
@@ -51,8 +56,8 @@ jobs:
     # Test
     #   This would typically be a build job when using workflows, possibly combined with build
     # This is based on your 1.0 configuration file or project settings
-    - run: sudo python ./setup.py develop && make check-2.7
-    - run: cd test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
+    - run: sudo python ./setup.py develop && make check-3.6
+    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
     # Teardown
     #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
     # Save test results

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [rocky]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.gitignore

@@ -10,6 +10,7 @@
 /.pytest_cache
 /.python-version
 /.tox
+.mypy_cache
 /.venv*
 /README
 /__pkginfo__.pyc

.travis.yml

@@ -1,7 +1,16 @@
 language: python
 
 python:
-  - 2.7 # this is a cheat here because travis doesn't do 2.4-2.6
+#  - '3.5'
+#  - '2.7'
+#  - '3.4'
+  - '3.6'
+  - '3.8'
+
+matrix:
+  include:
+    - python: '3.7'
+      dist: xenial  # required for Python >= 3.7 (travis-ci/travis-ci#9069)
 
 install:
 - pip install -e .

Makefile

@@ -40,9 +40,8 @@ check-3.0 check-3.1 check-3.2 check-3.6:
 check-3.7: pytest
 	$(MAKE) -C test check
 
-#:Tests for Python 2.4-2.5 (don't have pytest)
-check-2.4 check-2.5:
-	$(MAKE) -C test $@
+check-3.8:
+	$(MAKE) -C test check
 
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now

NEWS.md

@@ -1,11 +1,35 @@
+3.7.0: 2020-5-19 Primidi 1st Prairial - Alfalfa - HF
+====================================================
+
+The main impetus for this release is to pull in the recent changes from xdis.
+We simplify imports using xdis 4.6.0.
+
+There were some bugfixes to Python 3.4-3.8. See the ChangeLog for details
+
+
+3.6.7: 2020-4-27 xdis again
+===========================
+
+More upheaval in xdis which we need to track here.
+
+3.6.6: 2020-4-20 Love in the time of Cholera
+============================================
+
+The main reason for this release is an incompatablity bump in xdis which handles
+3.7 SipHash better.
+
+* Go over "yield" as an expression precidence
+* Some small alignment with code in decompyle3 for "or" and "and" was done
+
+
 3.6.5: 2020-4-1 April Fool
 ==========================
 
 Back port some of the changes in decompile3 here which mostly helps 3.7 and 3.8 decompilation, although this may also help 3.6ish versions too.
 
-- Handle nested `async for in for...`  and Better async comprehension detection via `xdis`.  Still more work is needed.
+- Handle nested `async for in for...`  and better async comprehension detection via `xdis`.  Still more work is needed.
 - include token number in listings when `-g` and there is a parser error
-- remove unneded `Makefile`s now that remake 4.3+1.5dbg is a thing that has `-c`
+- remove unneeded `Makefile`s now that remake 4.3+1.5dbg is a thing that has `-c`
 - Bug in finding annotations in functions with docstrings
 - Fix bug found by 2.4 sre_parse.py testing
 - Fix `transform` module's  `ifelseif` bugs

README.rst

@@ -89,7 +89,8 @@ This uses setup.py, so it follows the standard Python routine:
 
 ::
 
-    $ pip install -e .  # set up to run from source tree, or...
+    $ pip install -e .  # set up to run from source tree
+                        # Or if you want to install instead
     $ python setup.py install # may need sudo
 
 A GNU makefile is also provided so :code:`make install` (possibly as root or
@@ -100,7 +101,7 @@ Running Tests
 
 ::
 
-   $  make check
+   make check
 
 A GNU makefile has been added to smooth over setting running the right
 command, and running tests from fastest to slowest.
@@ -251,11 +252,14 @@ See Also
 .. _uncompyle2: https://github.com/wibiti/uncompyle2
 .. _unpyc37: https://github.com/andrew-tavera/unpyc37
 .. _this: https://github.com/rocky/python-uncompyle6/wiki/Deparsing-technology-and-its-use-in-exact-location-reporting
-.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg :target: https://travis-ci.org/rocky/python-uncompyle6
-.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg :target: https://repology.org/project/python:uncompyle6/versions
+.. |buildstatus| image:: https://travis-ci.org/rocky/python-uncompyle6.svg
+		 :target: https://travis-ci.org/rocky/python-uncompyle6
+.. |packagestatus| image:: https://repology.org/badge/vertical-allrepos/python:uncompyle6.svg
+		 :target: https://repology.org/project/python:uncompyle6/versions
 .. _PJOrion: http://www.koreanrandom.com/forum/topic/15280-pjorion-%D1%80%D0%B5%D0%B4%D0%B0%D0%BA%D1%82%D0%B8%D1%80%D0%BE%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5-%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%B4%D0%B5%D0%BA%D0%BE%D0%BC%D0%BF%D0%B8%D0%BB%D1%8F%D1%86%D0%B8%D1%8F-%D0%BE%D0%B1%D1%84
 .. _Deobfuscator: https://github.com/extremecoders-re/PjOrion-Deobfuscator
 .. _Py2EXE: https://en.wikipedia.org/wiki/Py2exe
 .. |Supported Python Versions| image:: https://img.shields.io/pypi/pyversions/uncompyle6.svg
-.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg :target: https://badge.fury.io/py/uncompyle6
+.. |Latest Version| image:: https://badge.fury.io/py/uncompyle6.svg
+		 :target: https://badge.fury.io/py/uncompyle6
 .. |Pypi Installs| image:: https://pepy.tech/badge/uncompyle6/month

__pkginfo__.py

@@ -21,6 +21,17 @@
 # less elegant than having it here with reduced code, albeit there
 # still is some room for improvement.
 
+# Python-version | package | last-version |
+# -----------------------------------------
+# 2.5            | pip     |  1.1         |
+# 2.6            | pip     |  1.5.6       |
+# 2.7            | pip     | 19.2.3       |
+# 2.7            | pip     |  1.2.1       |
+# 3.1            | pip     |  1.5.6       |
+# 3.2            | pip     |  7.1.2       |
+# 3.3            | pip     | 10.0.1       |
+# 3.4            | pip     | 19.1.1       |
+
 # Things that change more often go here.
 copyright   = """
 Copyright (C) 2015-2020 Rocky Bernstein <rb@dustyfeet.com>.
@@ -58,7 +69,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.2.4, < 4.3.0"]
+                      "xdis >= 4.6.0, < 4.7.0"]
 
 license            = "GPL3"
 mailing_list       = "python-debugger@googlegroups.com"

admin-tools/check-older-versions.sh

@@ -20,9 +20,8 @@ for version in $PYVERSIONS; do
 	exit $?
     fi
     make clean && python setup.py develop
-    if ! make check-short ; then
+    if ! make check ; then
 	exit $?
     fi
     echo === $version ===
 done
-make check

admin-tools/how-to-make-a-release.md

@@ -55,9 +55,8 @@
 # Make packages and tag
 
     $ . ./admin-tools/make-dist-older.sh
-	$ pyenv local 3.8.1
+	$ pyenv local 3.8.3
 	$ twine check dist/uncompyle6-$VERSION*
-    $ git tag release-python-2.4-$VERSION
     $ . ./admin-tools/make-dist-newer.sh
 	$ twine check dist/uncompyle6-$VERSION*
 

admin-tools/how-to-make-a-release.txt

@@ -1,46 +0,0 @@
-git pull
-
-Change version in uncompyle6/version.py
-source uncompyle6/version.py
-echo $VERSION
-git commit -m"Get ready for release $VERSION" .
-
-Update ChangeLog:
-  make ChangeLog
-
-Update NEWS from ChangeLog
-make check
-
-git commit --amend .
-
-git push
-
-Make sure pyenv is running
-# Pyenv
-
-source admin-tools/check-newer-versions.sh
-
-
-# Switch to python-2.4 and build that first...
-source admin-tools/setup-python-2.4
-
-rm ChangeLog
-git merge master
-
-Update NEWS from master branch
-
-git commit -m"Get ready for release $VERSION" .
-
-source admin-tools/check-older-versions.sh
-source admin-tools/check-newer-versions.sh
-
-make-dist-older.sh
-
-git tag release-python-2.4-$VERSION
-
-./make-dist-newer.sh
-
-git tag release-$VERSION
-
-
-twine upload dist/uncompyle6-${VERSION}*

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.9 3.6.10 2.6.9 3.3.7 2.7.17 3.2.6 3.1.5 3.4.10 3.7.7 3.8.2'
+export PYVERSIONS='3.5.9 3.6.10 2.6.9 3.3.7 2.7.18 3.2.6 3.1.5 3.4.10 3.7.7 3.8.3'

admin-tools/pyenv-older-versions

@@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='2.4.6 2.5.6'
+export PYVERSIONS='2.4.6 2.5.6 2.6.9'

admin-tools/setup-python-2.4.sh

@@ -15,4 +15,3 @@ cd $fulldir/..
     git checkout python-2.4 &&  pyenv local $PYTHON_VERSION && git pull
 cd $owd
 rm -v */.python-version || true
-pyenv local $PYTHON_VERSION

admin-tools/update-sources.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd $(dirname ${BASH_SOURCE[0]})/..
-git pull

appveyor.yml

@@ -53,6 +53,7 @@ install:
   # compiled extensions and are not provided as pre-built wheel packages,
   # pip will build them from source using the MSVC compiler matching the
   # target Python version and architecture
+  - "%CMD_IN_ENV% pip install git+git://github.com/rocky/python-uncompyle6.git#egg=uncompyle6-3.6.6"
   - "%CMD_IN_ENV% pip install -r requirements.txt"
 
 build_script:

pytest/test_deparse.py

@@ -30,7 +30,7 @@ def list_comp():
     [y for y in range(3)]
 
 def get_parsed_for_fn(fn):
-    code = fn.func_code
+    code = fn.__code__ if PYTHON3 else fn.func_code
     return deparse(code, version=PYTHON_VERSION)
 
 def check_expect(expect, parsed, fn_name):

pytest/test_fjt.py

@@ -20,7 +20,7 @@ def bug_loop(disassemble, tb=None):
     disassemble(tb)
 
 def test_if_in_for():
-    code = bug.func_code
+    code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
     if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         scan.build_instructions(code)

pytest/test_grammar.py

@@ -45,9 +45,6 @@ def test_grammar():
         expect_lhs.add("kvlist")
         expect_lhs.add("kv3")
         unused_rhs.add("dict")
-    else:
-        # NOTE: this may disappear
-        expect_lhs.add("except_handler_else")
 
         if PYTHON_VERSION < 3.7 and PYTHON_VERSION != 2.7:
             # NOTE: this may disappear

pytest/test_pysource.py-notyet

@@ -8,8 +8,12 @@ from uncompyle6.semantics.consts import (
 
 if PYTHON3:
     from io import StringIO
+    def iteritems(d):
+        return d.items()
 else:
     from StringIO import StringIO
+    def iteritems(d):
+        return d.iteritems()
 
 from uncompyle6.semantics.pysource import (SourceWalker, deparse_code2str)
 
@@ -26,7 +30,7 @@ def test_template_engine():
     # FIXME: and so on...
 
 from uncompyle6.semantics.consts import (
-    TABLE_R, TABLE_DIRECT,
+    TABLE_DIRECT, TABLE_R,
     )
 
 from uncompyle6.semantics.fragments import (
@@ -40,7 +44,7 @@ def test_tables():
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
-        for k, entry in t.iteritems():
+        for k, entry in iteritems(t):
             if k in skip_for_now:
                 continue
             fmt = entry[0]

pytest/test_single_compile.py

@@ -1,4 +1,7 @@
+import pytest
 from uncompyle6 import PYTHON_VERSION, code_deparse
+pytestmark = pytest.mark.skip(PYTHON_VERSION < 2.7,
+                              reason="need at least Python 2.7")
 
 if PYTHON_VERSION > 2.6:
     def test_single_mode():

pytest/test_token.py

@@ -1,14 +1,13 @@
-from uncompyle6 import PYTHON_VERSION
 from uncompyle6.scanners.tok import Token
 
 
 def test_token():
     # Test token formatting of: LOAD_CONST None
     t = Token("LOAD_CONST", offset=0, attr=None, pattr=None, has_arg=True)
-    expect = "                0  LOAD_CONST               None"
+    expect = "0  LOAD_CONST               None"
     # print(t.format())
     assert t
-    assert t.format() == expect
+    assert t.format().strip() == expect.strip()
 
     # Make sure equality testing of tokens ignores offset
     t2 = Token("LOAD_CONST", offset=2, attr=None, pattr=None, has_arg=True)
@@ -17,8 +16,8 @@ def test_token():
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
     t = Token("LOAD_CONST", offset=1, attr=False, pattr=False, has_arg=True)
-    expect = "                1  LOAD_CONST               False"
-    assert t.format() == expect
+    expect = "1  LOAD_CONST               False"
+    assert t.format().strip() == expect.strip()
 
 
 if __name__ == "__main__":

pytest/testdata/if-2.7.right

@@ -9,4 +9,4 @@
                12  JUMP_FORWARD          0  'to 15'
              15_0  COME_FROM            12  '12'
                15  LOAD_CONST               None
-               18  RETURN_VALUE
+               18  RETURN_VALUE     

pytest/testdata/ifelse-2.7.right

@@ -12,4 +12,4 @@
                18  STORE_NAME            2  'd'
              21_0  COME_FROM            12  '12'
                21  LOAD_CONST               None
-               24  RETURN_VALUE
+               24  RETURN_VALUE     

pytest/validate.py

@@ -12,8 +12,7 @@ import functools
 from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY, code_deparse
 
 # TODO : I think we can get xdis to support the dis api (python 3 version) by doing something like this there
-from xdis.bytecode import Bytecode
-from xdis.main import get_opcode
+from xdis import Bytecode, get_opcode
 
 opc = get_opcode(PYTHON_VERSION, IS_PYPY)
 Bytecode = functools.partial(Bytecode, opc=opc)

setup.py

@@ -1,43 +1,57 @@
 #!/usr/bin/env python
-"""Setup script for the 'uncompyle6' distribution."""
-
 import sys
 
+"""Setup script for the 'uncompyle6' distribution."""
+
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 4) <= SYS_VERSION  <= (2, 7)):
-    mess = "Python Release 2.4 .. 2.7 are supported in this code branch."
-    if ((3, 2) <= SYS_VERSION  <= (3, 8)):
-        mess += ("\nFor your Python, version %s, use the master code/branch." %
-                 sys.version[0:3])
-    else:
-        mess += ("\nThis package is not supported before Python 2.4. Your Python version is %s."
-                 % sys.version[0:3])
+if not ((2, 6) <= SYS_VERSION <= (3, 9)):
+    mess = "Python Release 2.6 .. 3.9 are supported in this code branch."
+    if (2, 4) <= SYS_VERSION <= (2, 7):
+        mess += (
+            "\nFor your Python, version %s, use the python-2.4 code/branch."
+            % sys.version[0:3]
+        )
+    elif SYS_VERSION < (2, 4):
+        mess += (
+            "\nThis package is not supported for Python version %s." % sys.version[0:3]
+        )
     print(mess)
     raise Exception(mess)
 
-from __pkginfo__ import \
-    author,           author_email,       install_requires,          \
-    license,          long_description,   classifiers,               \
-    entry_points,     modname,            py_modules,                \
-    short_desc,       VERSION,            web,                       \
-    zip_safe
+from __pkginfo__ import (
+    author,
+    author_email,
+    install_requires,
+    license,
+    long_description,
+    classifiers,
+    entry_points,
+    modname,
+    py_modules,
+    short_desc,
+    VERSION,
+    web,
+    zip_safe,
+)
 
 from setuptools import setup, find_packages
+
 setup(
-       author             = author,
-       author_email       = author_email,
-       classifiers        = classifiers,
-       description        = short_desc,
-       entry_points       = entry_points,
-       install_requires   = install_requires,
-       license            = license,
-       long_description   = long_description,
-       long_description_content_type = "text/x-rst",
-       name               = modname,
-       packages           = find_packages(),
-       py_modules         = py_modules,
-       test_suite         = 'nose.collector',
-       url                = web,
-       tests_require      = ['nose>=1.0'],
-       version            = VERSION,
-       zip_safe           = zip_safe)
+    author=author,
+    author_email=author_email,
+    classifiers=classifiers,
+    description=short_desc,
+    entry_points=entry_points,
+    install_requires=install_requires,
+    license=license,
+    long_description=long_description,
+    long_description_content_type="text/x-rst",
+    name=modname,
+    packages=find_packages(),
+    py_modules=py_modules,
+    test_suite="nose.collector",
+    url=web,
+    tests_require=["nose>=1.0"],
+    version=VERSION,
+    zip_safe=zip_safe,
+)

test-unit/test_grammar.py

@@ -1,55 +0,0 @@
-import re
-import unittest
-from uncompyle6 import PYTHON_VERSION, IS_PYPY # , PYTHON_VERSION
-from uncompyle6.parser import get_python_parser, python_parser
-
-class TestGrammar(unittest.TestCase):
-    def test_grammar(self):
-
-        def check_tokens(tokens, opcode_set):
-            remain_tokens = set(tokens) - opcode_set
-            remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
-            remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-            remain_tokens = set(remain_tokens) - opcode_set
-            self.assertEqual(remain_tokens, set([]),
-                    "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()))
-
-        p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-        (lhs, rhs, tokens,
-         right_recursive, dup_rhs) = p.check_sets()
-        expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
-        unused_rhs = set(['list', 'call', 'mkfunc',
-                          'mklambda',
-                          'unpack',])
-
-        expect_right_recursive = frozenset([('designList',
-                                             ('store', 'DUP_TOP', 'designList'))])
-        expect_lhs.add('kwarg')
-
-        self.assertEqual(expect_lhs, set(lhs))
-        self.assertEqual(unused_rhs, set(rhs))
-        self.assertEqual(expect_right_recursive, right_recursive)
-
-        expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                    ('LOAD_CONST',),
-                                    ('JUMP_BACK',), ('JUMP_FORWARD',)])
-
-        reduced_dup_rhs = {}
-        for k in dup_rhs:
-            if k not in expect_dup_rhs:
-                reduced_dup_rhs[k] = dup_rhs[k]
-                pass
-            pass
-        for k in reduced_dup_rhs:
-            print(k, reduced_dup_rhs[k])
-        # assert not reduced_dup_rhs, reduced_dup_rhs
-
-    def test_dup_rule(self):
-        import inspect
-        python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                      is_pypy=IS_PYPY,
-                      parser_debug={
-                          'dups': True, 'transition': False, 'reduce': False,
-                          'rules': False, 'errorstack': None, 'context': True})
-if __name__ == '__main__':
-    unittest.main()

test/.gitignore

@@ -1,3 +1,8 @@
 /.coverage
 /.python-version
 /nohup.out
+/pycdc
+/test_pycdc_tests.sh
+/test_uncompyle2.py
+/test_unpy33.py
+/test_unpy37.py

test/Makefile

@@ -30,7 +30,7 @@ check:
 	$(MAKE) check-$(PYTHON_VERSION)
 
 #: Run working tests from Python 2.6 or 2.7
-check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1  check-native-short
+check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-native-short
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
@@ -72,10 +72,10 @@ check-3.7: check-bytecode
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify $(COMPILE)
 
-# #: Run working tests from Python 3.8
-# check-3.8: check-bytecode
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
+#: Run working tests from Python 3.8
+check-3.8: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify $(COMPILE)
 
 # FIXME
 #: this is called when running under pypy3.5-5.8.0, pypy2-5.6.0, or pypy3.6-7.3.0
@@ -95,12 +95,19 @@ check-bytecode-2:
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
 
 #: Check deparsing bytecode 3.x only
+# check-bytecode-3:
+# 	$(PYTHON) test_pythonlib.py \
+#         --bytecode-3.0 \
+# 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+# 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+#         --bytecode-3.7  \
+# 	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
+
+# FIXME: Until we shaked out problems with xdis...
 check-bytecode-3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
-	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
+	$(PYTHON) test_pythonlib.py \
 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7 \
-	--bytecode-pypy3.2 --bytecode-pypy3.6
+        --bytecode-3.7  --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -166,7 +173,6 @@ check-bytecode-2.3:
 
 #: Check deparsing Python 2.4
 check-bytecode-2.4:
-	$(PYTHON) test_pythonlib.py --bytecode-2.4-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-2.4
 
 #: Check deparsing Python 2.5
@@ -295,20 +301,16 @@ check-bytecode-3.7:
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --syntax-verify
 
-# #: Check deparsing Python 3.8
-# check-bytecode-3.8:
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
-# 	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
+#: Check deparsing Python 3.8
+check-bytecode-3.8:
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --syntax-verify
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --syntax-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE)
 
-#: Run longer Python 2.6's lib files known to be okay
-check-2.4-ok:
-	$(PYTHON) test_pythonlib.py --ok-2.4 --verify $(COMPILE)
-
 #: Run longer Python 2.6's lib files known to be okay
 check-2.6-ok:
 	$(PYTHON) test_pythonlib.py --ok-2.6 --syntax-verify $(COMPILE)

test/dis-compare.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015, 2017 by Rocky Bernstein <rb@dustyfeet.com>
+# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
 #
+from __future__ import print_function
 
 
 import dis, os.path

test/run-and-email.sh

@@ -34,9 +34,10 @@ else
 fi
 
 MAIN="test_pyenvlib.py"
+
 USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-WHAT="uncompyle6 2.4 ${MAIN}"
+WHAT="uncompyle6 ${MAIN}"
 MAX_TESTS=${MAX_TESTS:-800}
 export BATCH=1
 
@@ -44,7 +45,7 @@ typeset -i RUN_STARTTIME=$(date +%s)
 
 # PYVERSIONS="3.5.6"
 MAILBODY=/tmp/${MAIN}-mailbody-$$.txt
-# for VERSION in 2.4.6 2.5.9 ; do
+# for VERSION in 3.3.7 ; do
 for VERSION in $PYVERSIONS ; do
     typeset -i rc=0
     LOGFILE=/tmp/${MAIN}-$VERSION-$$.log
@@ -66,7 +67,9 @@ for VERSION in $PYVERSIONS ; do
 	    MAX_TESTS=800
 	    ;;
 	3.6.10 )
-	    MAX_TESTS=1300  # about 2139 exist
+	    # MAX_TESTS=1300  # about 2139 exist
+	    # fails on _pyio.cpython-36.opt-1.pyc
+	    MAX_TESTS=34
 	    ;;
 	2.4.6 )
 	    MAX_TESTS=600

test/simple-decompile-code-test.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 
-from uncompyle6 import uncompyle
+from __future__ import print_function
+
 from uncompyle6.main import decompile
-from xdis.magics import sysinfo2float
+from xdis import sysinfo2float
 import sys, inspect
 
 def uncompyle_test():

test/simple_source/bug25/02_yield_bug.py

@@ -1,7 +0,0 @@
-# From 2.4 test_array.py
-# In Python 2.4 and earlier "yield" is not valid and instead
-# we must use "yield None". Bug was not adding "None"
-
-def yield_bug():
-    yield None
-    return

test/simple_source/bug25/04_try_else_confuse.py

@@ -1,19 +0,0 @@
-# From 2.4 test_sax.py
-# Bug was distinguishing try from try/else
-
-def verify_empty_attrs():
-    gvqk = 3
-    try:
-        gvk = 1/0
-    except ZeroDivisionError:
-        gvk = 1
-
-    try:
-        gvqk = 0
-    except KeyError:
-        gvqk = 1
-
-    # If try/else was used above the return will be 4
-    return gvk + gvqk
-
-assert 1 == verify_empty_attrs()

test/simple_source/bug33/02_pos_args.py

@@ -1,6 +1,8 @@
+# Python 3.3+
+#
 # From Python 3.3.6 hmac.py
 # Problem was getting wrong placement of positional args.
-# In 3.6+ paramter handling changes
+# In 3.6+ parameter handling changes
 
 # RUNNABLE!
 

test/simple_source/bug36/02_var_annotate.py

@@ -1,6 +1,9 @@
 # 3.6+ type annotations on variables
 from typing import List
 
+# This test program is part of the uncompyle6 test suite
+# tests STORE_ANNOTATION and SETUP_ANOTATIONS
+
 # RUNNABLE!
 y = 2
 x: bool

test/simple_source/bug37/04_async.py

@@ -1,23 +0,0 @@
-# from 3.7 test_contextlib_async.py
-# Bugs were not adding "async" when a function is a decorator,
-# and a misaligment when using "async with as".
-@_async_test
-async def test_enter(self):
-    self.assertIs(await manager.__aenter__(), manager)
-
-    async with manager as context:
-        async with woohoo() as x:
-            x = 1
-            y = 2
-        assert manager is context
-
-# From 3.7.6 test_coroutines.py
-# Bug was different form of code for "async with" below
-class CoroutineTest():
-    def test_with_8(self):
-        CNT = 0
-        async def foo():
-            nonlocal CNT
-            async with CM():
-                CNT += 1
-                return

test/simple_source/bug37/04_grammar.py

@@ -0,0 +1,87 @@
+# From test_grammar.py
+# RUNNABLE!
+def check_syntax_error(statement):
+    try:
+        compile(statement, '<test string>', 'exec')
+    except SyntaxError:
+        return
+    assert False
+
+def test_yield():
+    # Requires parentheses as call argument
+    def g():
+        f((yield 1), 1)
+    def g():
+        f((yield from ()))
+    def g():
+        f((yield from ()), 1)
+    def g():
+        f((yield 1))
+
+    # Allowed as standalone statement
+    def g():
+        yield 1
+
+    def g():
+        yield from ()
+
+    # Allowed as RHS of assignment
+    def g():
+        x = yield 1
+
+    def g():
+        x = yield from ()
+
+    # Ordinary yield accepts implicit tuples
+    def g():
+        yield 1, 1
+
+    def g():
+        x = yield 1, 1
+
+    # 'yield from' does not
+    check_syntax_error("def g(): yield from (), 1")
+    check_syntax_error("def g(): x = yield from (), 1")
+    # Requires parentheses as subexpression
+    def g():
+        1, (yield 1)
+
+    def g():
+        1, (yield from ())
+
+    check_syntax_error("def g(): 1, yield 1")
+    check_syntax_error("def g(): 1, yield from ()")
+    # Requires parentheses as call argument
+    def g():
+        f((yield 1))
+
+    def g():
+        f((yield 1), 1)
+
+    def g():
+        f((yield from ()))
+
+    def g():
+        f((yield from ()), 1)
+
+    check_syntax_error("def g(): f(yield 1)")
+    check_syntax_error("def g(): f(yield 1, 1)")
+    check_syntax_error("def g(): f(yield from ())")
+    check_syntax_error("def g(): f(yield from (), 1)")
+    # Not allowed at top level
+    check_syntax_error("yield")
+    check_syntax_error("yield from")
+    # Not allowed at class scope
+    check_syntax_error("class foo:yield 1")
+    check_syntax_error("class foo:yield from ()")
+    # Check annotation refleak on SyntaxError
+    check_syntax_error("def g(a:(yield)): pass")
+
+test_yield()
+
+# From test_types.py
+# Bug was needing parens around (yield 2)
+def gen_func():
+    yield 1
+    return (yield 2)
+gen = gen_func()

test/simple_source/bug37/10_async.py

@@ -0,0 +1,70 @@
+# from 3.7 test_contextlib_async.py
+# Bugs were not adding "async" when a function is a decorator,
+# and a misaligment when using "async with ... as".
+
+"""This program is self-checking!"""
+import asyncio
+from contextlib import asynccontextmanager, AbstractAsyncContextManager
+import functools
+
+
+def _async_test(func):
+    """Decorator to turn an async function into a test case."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        coro = func(*args, **kwargs)
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            return loop.run_until_complete(coro)
+        finally:
+            loop.close()
+            asyncio.set_event_loop(None)
+
+    return wrapper
+
+
+state = []
+
+
+@asynccontextmanager
+async def woohoo():
+    state.append(1)
+    yield 42
+    state.append(999)
+
+
+@_async_test
+async def test_enter():
+    class DefaultEnter(AbstractAsyncContextManager):
+        async def __aexit__(*args):
+            return
+            # await super().__aexit__(*args)
+
+    manager = DefaultEnter()
+    got_manager = await manager.__aenter__()
+    # print(got_manager, manager)
+    assert got_manager is manager
+
+    async with manager as context:
+        async with woohoo() as x:
+            x = 1
+            y = 2
+        assert manager is context
+
+
+# From 3.7.6 test_coroutines.py
+# Bug was different form of code for "async with" below
+class CoroutineTest:
+    def test_with_8(self):
+        CNT = 0
+
+        async def foo():
+            nonlocal CNT
+            async with CM():
+                CNT += 1
+                return
+
+
+test_enter()

test/stdlib/.gitignore

@@ -1 +1,3 @@
 /.python-version
+/runun33.sh
+/runun7.sh

test/stdlib/2.6-exclude.sh

@@ -1,4 +1,12 @@
 SKIP_TESTS=(
+    # ifelsestmt is borked in:
+    #   if filename == 'srcfile':
+    #      return srcfile
+    #   if filename == 'destfile':
+    #      return destfile
+    #   assert 0  # shouldn't reach here.
+    [test_shutil.py]=1
+
 
     [test___all__.py]=1  # it fails on its own
     [test___all__.py]=1 # it fails on its own
@@ -59,6 +67,7 @@ SKIP_TESTS=(
 
     [test_scriptpackages.py]=1 # it fails on its own
     [test_select.py]=1 # test takes too long to run: 11 seconds
+
     [test_socket.py]=1 # test takes too long to run: 12 seconds
     [test_startfile.py]=1 # it fails on its own
     [test_structmembers.py]=1 # it fails on its own

test/stdlib/3.6-exclude.sh

@@ -5,7 +5,6 @@ SKIP_TESTS=(
     [test_ftplib.py]=1 # FIXME: Works on c90ff51
     [test_slice.py]=1 # FIXME: Works on c90ff51
     [test_sort.py]=1 # FIXME: Works on c90ff51
-    [test_strftime.py]=1 # FIXME: Works on c90ff51
     [test_timeit.py]=1 # FIXME: Works on c90ff51
     [test_os.py]=1 # parse error FIXME: Works on c90ff51
 

test/stdlib/3.8-exclude.sh

@@ -1,10 +1,14 @@
 SKIP_TESTS=(
-    [test_time.py]=1 # FIXME: works on uncompyle6?
+    [test_mimetypes.py]=1 # parse error. decompile3 works. Release 3.6.4 works?
+    [test_time.py]=1 # FIXME: parse eror. decompyle3 works. Release 3.6.4 works?
+    [test_aifc.py]=1 # parse error; decompile3 works
+
+    [test_doctest2.py]=1 # test failures release 3.6.4 works?
+    [test_finalization.py]=1  # test failures release 3.6.4 works?
     [test_urllib2.py]=1 # FIXME: works on uncompyle6?
     [test_zipimport.py]=1 # FIXME: works on uncompyle6
 
     [test___all__.py]=1 # it fails on its own
-    [test_aifc.py]=1 # parse error
     [test_argparse.py]=1 #- it fails on its own
     [test_array.py]=1 #- parse error
     [test_asdl_parser.py]=1 # it fails on its own
@@ -76,6 +80,7 @@ SKIP_TESTS=(
     [test_difflib.py]=1   # parse error
     [test_dis.py]=1   # We change line numbers - duh!
     [test_doctest.py]=1 # parse error
+    [test_doctest2.py]=1 # test faiures relesae 3.6.4 works?
     [test_docxmlrpc.py]=1
     [test_dtrace.py]=1 # parse error
     [test_dummy_thread.py]=1 # parse error
@@ -91,6 +96,7 @@ SKIP_TESTS=(
     [test_exceptions.py]=1   # parse error
 
     [test_faulthandler.py]=1   # takes too long
+    [test_file_eintr.py]=1 # too long to run test; works on 3.7.7
     [test_fcntl.py]=1
     [test_filecmp.py]=1   # parse error
     [test_fileinput.py]=1

test/stdlib/runtests.sh

@@ -131,6 +131,7 @@ fulldir=$(pwd)
 
 # DECOMPILER=uncompyle2
 DECOMPILER=${DECOMPILER:-"$fulldir/../../bin/uncompyle6"}
+OPTS=${OPTS:-""}
 TESTDIR=/tmp/test${PYVERSION}
 if [[ -e $TESTDIR ]] ; then
     rm -fr $TESTDIR
@@ -139,6 +140,8 @@ fi
 PYENV_ROOT=${PYENV_ROOT:-$HOME/.pyenv}
 pyenv_local=$(pyenv local)
 
+echo Python version is $pyenv_local
+
 # pyenv version update
 for dir in ../ ../../ ; do
     cp -v .python-version $dir
@@ -147,18 +150,25 @@ done
 
 mkdir $TESTDIR || exit $?
 cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
-cd $TESTDIR/test
+if [[ $PYVERSION == 3.2 ]] ; then
+    cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR
+    cd $TESTDIR
+else
+    cd $TESTDIR/test
+fi
 pyenv local $FULLVERSION
 export PYTHONPATH=$TESTDIR
 export PATH=${PYENV_ROOT}/shims:${PATH}
 
+DONT_SKIP_TESTS=${DONT_SKIP_TESTS:-0}
+
 # Run tests
 typeset -i i=0
 typeset -i allerrs=0
 if [[ -n $1 ]] ; then
-    files=$1
-    typeset -a files_ary=( $(echo $1) )
-    if (( ${#files_ary[@]} == 1 )) ; then
+    files=$@
+    typeset -a files_ary=( $(echo $@) )
+    if (( ${#files_ary[@]} == 1 || DONT_SKIP_TESTS == 1 )) ; then
 	SKIP_TESTS=()
     fi
 else
@@ -190,7 +200,7 @@ for file in $files; do
     typeset -i ENDTIME=$(date +%s)
     typeset -i time_diff
     (( time_diff =  ENDTIME - STARTTIME))
-    if (( time_diff > 10 )) ; then
+    if (( time_diff > $timeout )) ; then
 	echo "Skipping test $file -- test takes too long to run: $time_diff seconds"
 	continue
     fi
@@ -202,7 +212,7 @@ for file in $files; do
     $fulldir/compile-file.py $file && \
     mv $file{,.orig} && \
     echo ==========  $(date +%X) Decompiling $file ===========
-    $DECOMPILER $decompiled_file > $file
+    $DECOMPILER $OPTS $decompiled_file > $file
     rc=$?
     if (( rc == 0 )) ; then
 	echo ========== $(date +%X) Running $file ===========

test/test_pyenvlib.py

@@ -20,7 +20,8 @@ Step 2: Run the test:
 	  test_pyenvlib --mylib --verify # decompile verify 'mylib'
 """
 
-from uncompyle6 import main, PYTHON3
+from __future__ import print_function
+
 import os, time, re, shutil, sys
 from fnmatch import fnmatch