Get ready for release 3.7.2

Hopefully by using first_child() we have something more robust now.

NEWS.md

@@ -1,3 +1,12 @@
+3.7.2: 2020-6-27
+================
+
+* Use newer xdis
+* Docstrings (again) which were broken again on earlier Python
+* Fix 2.6 and 2.7 decompilation bug in handling "list if" comprehensions
+
+
+
 3.7.1: 2020-6-12 Fleetwood66
 ====================================================
 

__pkginfo__.py

@@ -69,7 +69,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.6.1, < 4.8.0"]
+                      "xdis >= 4.7.0, <5.1.0"]
 
 license            = "GPL3"
 mailing_list       = "python-debugger@googlegroups.com"

admin-tools/how-to-make-a-release.md

@@ -2,17 +2,17 @@
 **Table of Contents**
 
 - [Get latest sources:](#get-latest-sources)
-- [Change version in uncompyle6/version.py](#change-version-in-uncompyle6versionpy)
+- [Change version in uncompyle6/version.py:](#change-version-in-uncompyle6versionpy)
 - [Update ChangeLog:](#update-changelog)
-- [Update NEWS from ChangeLog:](#update-news-from-changelog)
+- [Update NEWS.md from ChangeLog:](#update-newsmd-from-changelog)
 - [Make sure pyenv is running and check newer versions](#make-sure-pyenv-is-running-and-check-newer-versions)
 - [Switch to python-2.4, sync that up and build that first since it creates a tarball which we don't want.](#switch-to-python-24-sync-that-up-and-build-that-first-since-it-creates-a-tarball-which-we-dont-want)
-- [Update NEWS from master branch](#update-news-from-master-branch)
+- [Check against older versions](#check-against-older-versions)
-- [Check against all versions](#check-against-all-versions)
 - [Make packages and tag](#make-packages-and-tag)
-- [Upload single package and look at Rst Formating](#upload-single-package-and-look-at-rst-formating)
+- [Check package on github](#check-package-on-github)
-- [Upload rest of versions](#upload-rest-of-versions)
+- [Release on Github](#release-on-github)
-- [Push tags:](#push-tags)
+- [Get onto PyPI](#get-onto-pypi)
+- [Update tags:](#update-tags)
 
 <!-- markdown-toc end -->
 # Get latest sources:
@@ -60,27 +60,36 @@
     $ . ./admin-tools/make-dist-newer.sh
 	$ twine check dist/uncompyle6-$VERSION*
 
-# Upload single package and look at Rst Formating
+# Check package on github
 
-	$ twine check dist/uncompyle6-${VERSION}*
+	$ mkdir /tmp/gittest; pushd /tmp/gittest
-    $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
+	$ pyenv local 3.7.5
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
 
-# Upload rest of versions
+# Release on Github
-
-    $ twine upload dist/uncompyle6-${VERSION}*
 
 Goto https://github.com/rocky/python-uncompyle6/releases
 
-# Push tags:
+Now check the *tagged* release. (Checking the untagged release was previously done).
+
+Todo: turn this into a script in `admin-tools`
+
+	$ pushd /tmp/gittest
+	$ pip install -e git://github.com/rocky/python-uncompyle6.git@$VERSION#egg=uncompyle6
+	$ uncompyle6 --help
+	$ pip uninstall uncompyle6
+	$ popd
+
+
+# Get onto PyPI
+
+    $ twine upload dist/uncompyle6-${VERSION}*
+
+
+# Update tags:
 
     $ git push --tags
-
+    $ git pull --tags
-# Check on a VM
-
-    $ cd /virtual/vagrant/virtual/vagrant/ubuntu-zesty
-	$ vagrant up
-	$ vagrant ssh
-	$ pyenv local 3.5.2
-	$ pip install --upgrade uncompyle6
-	$ exit
-	$ vagrant halt

test/run-and-email.sh

@@ -51,7 +51,7 @@ for VERSION in $PYVERSIONS ; do
     LOGFILE=/tmp/${MAIN}-$VERSION-$$.log
 
     case "$VERSION" in
-	3.7.7 | 3.8.2 | 3.1.5 | 3.0.1 )
+	3.7.7 | 3.8.3 | 3.1.5 | 3.0.1 )
 	    continue
 	    ;;
 	3.5.9 )

test/stdlib/2.5-exclude.sh

@@ -55,7 +55,6 @@ SKIP_TESTS=(
     [test_ossaudiodev.py]=1 # it fails on its own
     [test_pdb.py]=1        # Line-number specific
     [test_pep277.py]=1    # it fails on its own
-    [test_pep352.py]=1     # Investigate
     [test_plistlib.py]=1 # it fails on its own
     [test_pwd.py]=1 # Long test - might work? Control flow?
     [test_pyclbr.py]=1 # Investigate

test/stdlib/2.6-exclude.sh

@@ -60,7 +60,6 @@ SKIP_TESTS=(
     [test_ossaudiodev.py]=1 # it fails on its own
 
     [test_pep277.py]=1 # it fails on its own
-    [test_pep352.py]=1     # Investigate
     [test_pyclbr.py]=1 # Investigate
     [test_pwd.py]=1 # Long test - might work? Control flow?
     [test_py3kwarn.py]=1 # it fails on its own

test/stdlib/3.7-exclude.sh

@@ -63,6 +63,7 @@ SKIP_TESTS=(
 
     [test_faulthandler.py]=1   # test takes too long before decompiling
     [test_fileinput.py]=1 # Test assertion failures
+    [test_finalization.py]=1 # if/else logic
     [test_frame.py]=1 # test assertion errors
     [test_ftplib.py]=1 # parse error
     [test_fstring.py]=1 # need to disambiguate leading fstrings from docstrings

uncompyle6/main.py

@@ -119,12 +119,10 @@ def decompile(
                 mapstream = _get_outstream(mapstream)
 
             deparsed = deparse_code_with_map(
-                bytecode_version,
                 co,
                 out,
-                showasm,
+                bytecode_version,
-                showast,
+                debug_opts,
-                showgrammar,
                 code_objects=code_objects,
                 is_pypy=is_pypy,
             )

uncompyle6/parsers/parse26.py

@@ -232,7 +232,10 @@ class Python26Parser(Python2Parser):
 
         comp_for ::= SETUP_LOOP expr for_iter store comp_iter jb_pb_come_from
 
-        comp_body ::= gen_comp_body
+        comp_iter   ::= comp_if_not
+        comp_if_not ::= expr jmp_true comp_iter
+
+        comp_body   ::= gen_comp_body
 
         for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
 

uncompyle6/parsers/parse27.py

@@ -8,7 +8,6 @@ from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse2 import Python2Parser
 from uncompyle6.parsers.reducecheck import (
     or_check,
-    ifelsestmt,
     tryelsestmt,
 )
 

uncompyle6/parsers/parse38.py

@@ -45,6 +45,7 @@ class Python38Parser(Python37Parser):
         stmt               ::= try_elsestmtl38
         stmt               ::= try_except_ret38
         stmt               ::= try_except38
+        stmt               ::= try_except_as
         stmt               ::= whilestmt38
         stmt               ::= whileTruestmt38
         stmt               ::= call_stmt
@@ -133,6 +134,8 @@ class Python38Parser(Python37Parser):
         except_cond1       ::= DUP_TOP expr COMPARE_OP jmp_false
                                POP_TOP POP_TOP POP_TOP
                                POP_EXCEPT
+        except_cond_as     ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
+                               POP_TOP STORE_FAST POP_TOP
 
         try_elsestmtl38    ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
                                except_handler38 COME_FROM
@@ -145,6 +148,10 @@ class Python38Parser(Python37Parser):
         # suite_stmts has a return
         try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
                                except_handler38b
+        try_except_as      ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                               except_handler_as END_FINALLY COME_FROM
+        try_except_as      ::= SETUP_FINALLY suite_stmts
+                               except_handler_as END_FINALLY COME_FROM
 
         try_except_ret38   ::= SETUP_FINALLY returns except_ret38a
         try_except_ret38a  ::= SETUP_FINALLY returns except_handler38c
@@ -165,6 +172,11 @@ class Python38Parser(Python37Parser):
         except_handler38a  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
                                POP_EXCEPT POP_TOP stmts END_FINALLY
 
+        except_handler38c  ::= COME_FROM_FINALLY except_cond1a except_stmts
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+        except_handler_as  ::= COME_FROM_FINALLY except_cond_as tryfinallystmt
+                               POP_EXCEPT JUMP_FORWARD COME_FROM
+
         tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
                                BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
                                END_FINALLY

uncompyle6/parsers/reducecheck/ifelsestmt.py

@@ -105,6 +105,24 @@ IFELSE_STMT_RULES = frozenset(
                 "opt_come_from_except",
             ),
         ),
+        (
+            "ifelsestmt",
+            (
+                "testexpr",
+                "stmts",
+                "jf_cfs",
+                "\\e_else_suite_opt",
+                "\\e_opt_come_from_except")
+        ),
+        (
+            "ifelsestmt",
+            (
+                "testexpr",
+                "stmts",
+                "jf_cfs",
+                "\\e_else_suite_opt",
+                "opt_come_from_except")
+        ),
     ])
 
 def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
@@ -113,6 +131,11 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
         # ifelsestmt jumped outside of loop. No good.
         return True
 
+    # print("XXX", first, last)
+    # for t in range(first, last):
+    #     print(tokens[t])
+    # print("=" * 30)
+
     if rule not in IFELSE_STMT_RULES:
         # print("XXX", rule)
         return False
@@ -186,9 +209,7 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
             if jump_else_end == "jf_cf_pop":
                 jump_else_end = jump_else_end[0]
 
-            jump_to_jump = False
             if jump_else_end == "JUMP_FORWARD":
-                jump_to_jump = True
                 endif_target = int(jump_else_end.pattr)
                 last_offset = tokens[last].off2int()
                 if endif_target != last_offset:

uncompyle6/semantics/customize38.py

@@ -50,12 +50,24 @@ def customize_for_version38(self, version):
             "%|%c\n", 0
         ),
 
+        "except_cond_as": (
+            "%|except %c as %c:\n",
+            (1, "expr"),
+            (-2, "STORE_FAST"),
+            ),
+
         'except_handler38': (
             '%c', (2, 'except_stmts') ),
 
         'except_handler38a': (
             '%c', (-2, 'stmts') ),
 
+        "except_handler_as": (
+            "%c%+\n%+%c%-",
+            (1, "except_cond_as"),
+            (2, "tryfinallystmt"),
+        ),
+
         'except_ret38a': (
             'return %c', (4, 'expr') ),
 
@@ -105,6 +117,13 @@ def customize_for_version38(self, version):
         'try_except38': (
             '%|try:\n%+%c\n%-%|except:\n%|%-%c\n\n',
                    (-2, 'suite_stmts_opt'), (-1, 'except_handler38a') ),
+
+        "try_except_as": (
+            "%|try:\n%+%c%-\n%|%-%c\n\n",
+            (-4, "suite_stmts"),  # Go from the end because of POP_BLOCK variation
+            (-3, "except_handler_as"),
+        ),
+
         "try_except_ret38": (
             "%|try:\n%+%c%-\n%|except:\n%+%|%c%-\n\n",
             (1, "returns"),

uncompyle6/semantics/pysource.py

@@ -895,6 +895,12 @@ class SourceWalker(GenericASTTraversal, object):
         doc_node = node[0]
         if doc_node.attr:
             docstring = doc_node.attr
+            if not isinstance(docstring, str):
+                # FIXME: we have mistakenly tagged something as a doc
+                # string in transform when it isn't one.
+                # The rule in n_mkfunc is pretty flaky.
+                self.prune()
+                return
         else:
             docstring = node[0].pattr
 

uncompyle6/semantics/transform.py

@@ -95,10 +95,18 @@ class TreeTransform(GenericASTTraversal, object):
 
         code = find_code_node(node, code_index).attr
 
+        mkfunc_pattr = node[-1].pattr
+        if isinstance(mkfunc_pattr, tuple):
+            assert len(mkfunc_pattr, 4) and isinstance(mkfunc_pattr, int)
+            is_closure = node[-1].pattr[3] != 0
+        else:
+            # FIXME: This is what we had before. It is hoaky and probably wrong.
+            is_closure = mkfunc_pattr == "closure"
+
         if (
-            node[-1].pattr != "closure"
+            (not is_closure)
             and len(code.co_consts) > 0
-            and code.co_consts[0] is not None
+            and isinstance(code.co_consts[0], str)
         ):
             docstring_node = SyntaxTree(
                 "docstring", [Token("LOAD_STR", has_arg=True, pattr=code.co_consts[0])]
@@ -433,6 +441,7 @@ class TreeTransform(GenericASTTraversal, object):
                     ast[i] = ast[i][0]
 
                 if is_docstring(self.ast[i]):
+                    load_const = self.ast[i].first_child()
                     docstring_ast = SyntaxTree(
                         "docstring",
                         [
@@ -440,8 +449,8 @@ class TreeTransform(GenericASTTraversal, object):
                                 "LOAD_STR",
                                 has_arg=True,
                                 offset=0,
-                                attr=self.ast[i][0][0].attr,
+                                attr=load_const.attr,
-                                pattr=self.ast[i][0][0].pattr,
+                                pattr=load_const.pattr,
                             )
                         ],
                     )

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside POSIX shell as
 # well as importing into Python
-VERSION="3.7.1"  # noqa
+VERSION="3.7.2"  # noqa