Merge branch 'master' into python-2.4

NEWS.md

@@ -1,3 +1,12 @@
+3.7.1: 2020-6-12 Fleetwood66
+====================================================
+
+Released to pick up new xdis version which has fixes to read bytestings better on 3.x
+
+* Handle 3.7+ "else" branch removal adAs seen in `_cmp()` of `python3.8/distutils/version.py` with optimization `-O2`
+* 3.6+ "with" and "with .. as" grammar improvements
+* ast-check for "for" loop was missing some grammar rules
+
 3.7.0: 2020-5-19 Primidi 1st Prairial - Alfalfa - HF
 ====================================================
 

README.rst

@@ -210,7 +210,7 @@ however that the magic of a released version is usually the same as
 the *last* candidate version prior to release.
 
 There are also customized Python interpreters, notably Dropbox,
-which use their own magic and encrypt bytcode. With the exception of
+which use their own magic and encrypt bytecode. With the exception of
 the Dropbox's old Python 2.5 interpreter this kind of thing is not
 handled.
 
@@ -229,7 +229,7 @@ There is lots to do, so please dig in and help.
 See Also
 --------
 
-* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back ehre.
+* https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back here.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
 * https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There are situations where :code:`uncompyle6` results are incorrect while :code:`uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because :code:`uncompyle6` adheres to accuracy over idiomatic Python, :code:`uncompyle2` can produce more natural-looking code when it is correct. Currently :code:`uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details

__pkginfo__.py

@@ -69,7 +69,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.6.0, < 4.7.0"]
+                      "xdis >= 4.6.1, < 4.8.0"]
 
 license            = "GPL3"
 mailing_list       = "python-debugger@googlegroups.com"

admin-tools/how-to-make-a-release.md

@@ -58,7 +58,6 @@
 	$ pyenv local 3.8.3
 	$ twine check dist/uncompyle6-$VERSION*
     $ git tag release-python-2.4-$VERSION
-	$ twine check dist/uncompyle6-$VERSION*
     $ . ./admin-tools/make-dist-newer.sh
 
 # Upload single package and look at Rst Formating

test/add-test.py

@@ -19,6 +19,7 @@ for path in py_source:
     else:
         cfile = "bytecode_%s%s/%s" % (version, suffix, short) + "c"
     print("byte-compiling %s to %s" % (path, cfile))
-    py_compile.compile(path, cfile)
+    optimize = 2
+    py_compile.compile(path, cfile, optimize=optimize)
     if isinstance(version, str) or version >= (2, 6, 0):
         os.system("../bin/uncompyle6 -a -T %s" % cfile)

test/simple_source/bug35/04_call_function.py

@@ -2,6 +2,7 @@
 # Bug was code not knowing which Python versions
 # have kwargs coming before positional args in code.
 
+"""This program is self-checking!"""
 # RUNNABLE!
 
 def tometadata(self, metadata, schema, Table, args, name=None):

test/simple_source/bug37/03_else_removal.py

@@ -0,0 +1,12 @@
+# From python3.8/distutils/version.py with optimization -O2
+# The bug was that the other "else" constant propagated removed.
+
+# NOTE: this program needs to be compile with optimization
+def _cmp (b, c):
+    if b:
+        if c:
+            return 0
+        else:
+            return 1
+    else:
+        assert False, "never get here"

test/simple_source/stmts/04_withas.py

@@ -16,3 +16,25 @@ def withas_bug(self, nested, a, b):
     with self.assertRaises(ZeroDivisionError):
         with nested(a(), b()) as (x, y):
                 1 // 0
+
+# From 3.7.7 test_functools.py
+# Bug is a unreachable code after "return"
+def test_invalid_registrations(x):
+    return
+    with x:
+        x = 1
+
+# From 3.7.7 test_re.py
+# Bug was hooking in c_with.
+def test_re_tests(tests):
+    for t in tests:
+        with a:
+            continue
+
+# Adapted from 3.8 distutils/command/config.py
+# In 3.8 the problem was in handling "with .. as" code
+def _gen_temp_sourcefile(x, a, headers, lang):
+    with x as y:
+        if a:
+            y = 2
+    return 5

test/stdlib/3.2-exclude.sh

@@ -17,8 +17,6 @@ SKIP_TESTS=(
     [test_peepholer.py]=1
     [test_pep352.py]=1
 
-    [test_quopri.py]=1 # TypeError: Can't convert 'bytes' object to str implicitly
-
     [test_runpy.py]=1
 
     [test_ssl.py]=1 # too installation specific
@@ -35,5 +33,4 @@ if (( BATCH )) ; then
     # Fails in crontab environment?
     # Figure out what's up here
     SKIP_TESTS[test_exception_variations.py]=1
-    SKIP_TESTS[test_quopri.py]=1
 fi

test/stdlib/3.3-exclude.sh

@@ -1,4 +1,15 @@
 SKIP_TESTS=(
+    # FIXME: Did this work sometime in the past ?
+    #   for elem in g(s):
+    #     if not tgt and isOdd(elem): continue
+    # is erroneously:
+    #   for elem in g(s):
+    #    if tgt or isOdd(elem):
+    #       pass
+    #    else:
+    #       tgt.append(elem)
+    [test_itertools.py]=1
+
     [test_buffer.py]=1  # FIXME: Works on c90ff51
     [test_cmath.py]=1  # FIXME: Works on c90ff51
 

test/stdlib/3.4-exclude.sh

@@ -1,4 +1,15 @@
 SKIP_TESTS=(
+    # FIXME: Did this work sometime in the past ?
+    #   for elem in g(s):
+    #     if not tgt and isOdd(elem): continue
+    # is erroneously:
+    #   for elem in g(s):
+    #    if tgt or isOdd(elem):
+    #       pass
+    #    else:
+    #       tgt.append(elem)
+    [test_itertools.py]=1
+
     [test_buffer.py]=1  # FIXME: Works on c90ff51
     [test_cmath.py]=1  # FIXME: Works on c90ff51
     [test_strftime.py]=1  # FIXME: Works on c90ff51
@@ -87,5 +98,4 @@ if (( batch )) ; then
     # Figure out what's up here
     SKIP_TESTS[test_exception_variations.py]=1
     SKIP_TESTS[test_mailbox.py]=1 # Takes to long on POWER; over 15 secs
-    SKIP_TESTS[test_quopri.py]=1
 fi

test/stdlib/3.5-exclude.sh

@@ -138,7 +138,6 @@ if (( BATCH )) ; then
     SKIP_TESTS[test_ioctl.py]=1 # it fails on its own
     SKIP_TESTS[test_poplib.py]=1 # May be a result of POWER installation
 
-    SKIP_TESTS[test_quopri.py]=1
     SKIP_TESTS[test_sysconfig.py]=1 # POWER extension fails
     SKIP_TESTS[test_tarfile.py]=1 # too long to run on POWER 15 secs
     SKIP_TESTS[test_venv.py]=1 # takes too long 11 seconds

test/stdlib/3.6-exclude.sh

@@ -124,8 +124,6 @@ SKIP_TESTS=(
     [test_pyclbr.py]=1 # it fails on its own
     [test_pydoc.py]=1 # it fails on its own
 
-    [test_quopri.py]=1      # AssertionError: b'123=four' != '123=four'
-
     [test_random.py]=1 # it fails on its own
     [test_range.py]=1
     [test_regrtest.py]=1 # test takes too long to run: 12 seconds

test/stdlib/3.7-exclude.sh

@@ -1,4 +1,19 @@
 SKIP_TESTS=(
+    # FIXME: Did this work sometime in the past ?
+    #   for elem in g(s):
+    #     if not tgt and isOdd(elem): continue
+    # is erroneously:
+    #   for elem in g(s):
+    #    if tgt or isOdd(elem):
+    #       pass
+    #    else:
+    #       tgt.append(elem)
+    [test_itertools.py]=1
+
+    # Fails on decompyle3 as well.
+    # complicated control flow and "and/or" expressions
+    [test_pickle.py]=1
+
     [test_builtin.py]=1 # FIXME works on decompyle6
     [test_context.py]=1 # FIXME works on decompyle6
     [test_doctest2.py]=1 # FIXME works on decompyle6

test/stdlib/runtests.sh

@@ -78,7 +78,6 @@ case $PYVERSION in
 	    # Fails in crontab environment?
 	    # Figure out what's up here
 	    SKIP_TESTS[test_exception_variations.py]=1
-	    SKIP_TESTS[test_quopri.py]=1
 	fi
 	;;
     3.1)
@@ -91,7 +90,6 @@ case $PYVERSION in
 	    # Fails in crontab environment?
 	    # Figure out what's up here
 	    SKIP_TESTS[test_exception_variations.py]=1
-	    SKIP_TESTS[test_quopri.py]=1
 	fi
 	;;
     3.2)

uncompyle6/parsers/parse3.py

@@ -681,6 +681,7 @@ class Python3Parser(PythonParser):
                 "RAISE",
                 "SETUP",
                 "UNPACK",
+                "WITH",
             )
         )
 

uncompyle6/parsers/parse36.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2019 Rocky Bernstein
+#  Copyright (c) 2016-2020 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -304,28 +304,25 @@ class Python36Parser(Python35Parser):
                 self.addRule(rule, nop_func)
                 # Check to combine assignment + annotation into one statement
                 self.check_reduce['assign'] = 'token'
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
             elif opname == 'SETUP_WITH':
                 rules_str = """
-                with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
-                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                  with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
+                                 with_suffix
 
-                # Removes POP_BLOCK LOAD_CONST from 3.6-
-                withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
-                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                  # Removes POP_BLOCK LOAD_CONST from 3.6-
+                  withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                                 with_suffix
+                  with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                 BEGIN_FINALLY COME_FROM_WITH
+                                 with_suffix
                 """
-                if self.version < 3.8:
-                    rules_str += """
-                    with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
-                                   LOAD_CONST
-                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
-                    """
-                else:
-                    rules_str += """
-                    with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
-                                   BEGIN_FINALLY COME_FROM_WITH
-                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH
-                                   END_FINALLY
-                    """
                 self.addRule(rules_str, nop_func)
                 pass
             pass

uncompyle6/parsers/parse37.py

@@ -93,6 +93,9 @@ class Python37Parser(Python37BaseParser):
         else_suitec ::= c_stmts
         else_suitec ::= returns
 
+        else_suite_opt ::= else_suite
+        else_suite_opt ::= pass
+
         stmt ::= classdef
         stmt ::= call_stmt
 
@@ -634,6 +637,12 @@ class Python37Parser(Python37BaseParser):
         if_exp37                   ::= expr expr jf_cfs expr COME_FROM
         jf_cfs                     ::= JUMP_FORWARD _come_froms
         ifelsestmt                 ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except
+
+        # This is probably more realistically an "ifstmt" (with a null else)
+        # see _cmp() of python3.8/distutils/__pycache__/version.cpython-38.opt-1.pyc
+        ifelsestmt                 ::= testexpr stmts jf_cfs else_suite_opt opt_come_from_except
+
+
         expr_pjit                  ::= expr POP_JUMP_IF_TRUE
         expr_jit                   ::= expr JUMP_IF_TRUE
         expr_jt                    ::= expr jmp_true

uncompyle6/parsers/parse37base.py

@@ -127,6 +127,7 @@ class Python37BaseParser(PythonParser):
                 "RAISE",
                 "SETUP",
                 "UNPACK",
+                "WITH",
             )
         )
 
@@ -993,55 +994,70 @@ class Python37BaseParser(PythonParser):
                 )
                 custom_ops_processed.add(opname)
 
+            elif opname == "WITH_CLEANUP_START":
+                rules_str = """
+                  stmt        ::= with_null
+                  with_null   ::= with_suffix
+                  with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                """
+                self.addRule(rules_str, nop_func)
             elif opname == "SETUP_WITH":
                 rules_str = """
                   stmt       ::= with
                   stmt       ::= withasstmt
 
-                  with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
-                                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                  with       ::= expr
+                                 SETUP_WITH POP_TOP
+                                 suite_stmts_opt
+                                 COME_FROM_WITH
+                                 with_suffix
                   withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
-                                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                                 with_suffix
+
+                  with       ::= expr
+                                 SETUP_WITH POP_TOP
+                                 suite_stmts_opt
+                                 POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                 with_suffix
+
+                  withasstmt ::= expr
+                                 SETUP_WITH store suite_stmts_opt
+                                 POP_BLOCK LOAD_CONST COME_FROM_WITH
+                                 with_suffix
 
                   with       ::= expr
                                  SETUP_WITH POP_TOP suite_stmts_opt
                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
-                                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                                 with_suffix
                   withasstmt ::= expr
                                  SETUP_WITH store suite_stmts_opt
                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
-                                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
-
-                  with       ::= expr
-                                 SETUP_WITH POP_TOP suite_stmts_opt
-                                 POP_BLOCK LOAD_CONST COME_FROM_WITH
-                                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
-                  withasstmt ::= expr
-                                 SETUP_WITH store suite_stmts_opt
-                                 POP_BLOCK LOAD_CONST COME_FROM_WITH
-                                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                                 with_suffix
                 """
                 if self.version < 3.8:
                     rules_str += """
                     with     ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
                                    LOAD_CONST
-                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                                   with_suffix
                     """
                 else:
                     rules_str += """
-                      with    ::= expr
+                      with       ::= expr
                                      SETUP_WITH POP_TOP suite_stmts_opt
                                      POP_BLOCK LOAD_CONST COME_FROM_WITH
-                                     WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                                     with_suffix
 
                       withasstmt ::= expr
                                      SETUP_WITH store suite_stmts_opt
                                      POP_BLOCK LOAD_CONST COME_FROM_WITH
 
-                       with    ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                      withasstmt ::= expr
+                                     SETUP_WITH store suite_stmts
+                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
+
+                      with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
                                      BEGIN_FINALLY COME_FROM_WITH
-                                     WITH_CLEANUP_START WITH_CLEANUP_FINISH
-                                     END_FINALLY
+                                     with_suffix
                     """
                 self.addRule(rules_str, nop_func)
 

uncompyle6/parsers/reducecheck/ifelsestmt.py

@@ -95,6 +95,16 @@ IFELSE_STMT_RULES = frozenset(
                 "else_suite",
             ),
         ),
+        (
+            "ifelsestmt",
+            (
+                "testexpr",
+                "stmts",
+                "jf_cfs",
+                "else_suite_opt",
+                "opt_come_from_except",
+            ),
+        ),
     ])
 
 def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
@@ -108,7 +118,7 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
         return False
 
     # Avoid if/else where the "then" is a "raise_stmt1" for an
-    # assert statemetn. Parse this as an "assert" instead.
+    # assert statement. Parse this as an "assert" instead.
     stmts = ast[1]
     if stmts in ("c_stmts",) and len(stmts) == 1:
         raise_stmt1 = stmts[0]

uncompyle6/semantics/check_ast.py

@@ -8,22 +8,29 @@ FIXME idea: extend parsing system to do same kinds of checks or nonterminal
 before reduction and don't reduce when there is a problem.
 """
 
+
 def checker(ast, in_loop, errors):
     if ast is None:
         return
-    in_loop = (in_loop or (ast.kind in ('while1stmt', 'whileTruestmt',
-                                      'whilestmt', 'whileelsestmt', 'while1elsestmt',
-                                      'for_block'))
-               or ast.kind.startswith('async_for'))
-    if ast.kind in ('aug_assign1', 'aug_assign2') and ast[0][0] == 'and':
+    in_loop = (
+        in_loop
+        or ast.kind.startswith("for")
+        or ast.kind.startswith("while")
+        or ast.kind.startswith("async_for")
+    )
+    if ast.kind in ("aug_assign1", "aug_assign2") and ast[0][0] == "and":
         text = str(ast)
-        error_text = '\n# improper augmented assigment (e.g. +=, *=, ...):\n#\t' + '\n# '.join(text.split("\n")) + '\n'
+        error_text = (
+            "\n# improper augmented assigment (e.g. +=, *=, ...):\n#\t"
+            + "\n# ".join(text.split("\n"))
+            + "\n"
+        )
         errors.append(error_text)
 
     for node in ast:
-        if not in_loop and node.kind in ('continue', 'break'):
+        if not in_loop and node.kind in ("continue", "break"):
             text = str(node)
-            error_text = '\n# not in loop:\n#\t' + '\n# '.join(text.split("\n"))
+            error_text = "\n# not in loop:\n#\t" + "\n# ".join(text.split("\n"))
             errors.append(error_text)
-        if hasattr(node, '__repr1__'):
+        if hasattr(node, "__repr1__"):
             checker(node, in_loop, errors)

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside POSIX shell as
 # well as importing into Python
-VERSION="3.7.0"  # noqa
+VERSION="3.7.1"  # noqa