Merge branch 'master' into python-3.3-to-3.5

admin-tools/pyenv-newest-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.17'
+export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.18'

admin-tools/setup-master.sh

@@ -18,14 +18,9 @@ function checkout_version {
     return $?
 }
 
-# FIXME put some of the below in a common routine
-function finish {
-  cd $owd
-}
 owd=$(pwd)
-trap finish EXIT
 
-export PATH=$HOME/.pyenv/bin/pyenv:$PATHb
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
 
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
@@ -35,4 +30,4 @@ cd $fulldir/..
 
 git pull
 rm -v */.python-version || true
-finish
+cd $owd

admin-tools/setup-python-2.4.sh

@@ -19,11 +19,7 @@ function checkout_version {
     return $?
 }
 
-function finish {
-  cd $owd
-}
 owd=$(pwd)
-trap finish EXIT
 
 export PATH=$HOME/.pyenv/bin/pyenv:$PATH
 
@@ -34,4 +30,4 @@ fulldir=$(readlink -f $mydir)
 
 git pull
 rm -v */.python-version || true
-finish
+cd $owd

admin-tools/setup-python-3.0.sh

@@ -20,9 +20,6 @@ function checkout_version {
     return $?
 }
 
-function finish {
-  cd $owd
-}
 owd=$(pwd)
 trap finish EXIT
 
@@ -36,4 +33,4 @@ cd $fulldir/..
 
 git pull
 rm -v */.python-version || true
-finish
+cd $owd

admin-tools/setup-python-3.3.sh

@@ -19,11 +19,7 @@ function checkout_version {
     return $?
 }
 
-function finish {
-  cd $owd
-}
 owd=$(pwd)
-trap finish EXIT
 
 export PATH=$HOME/.pyenv/bin/pyenv:$PATH
 
@@ -36,4 +32,4 @@ rm -v */.python-version || true
 
 git pull
 rm -v */.python-version || true
-finish
+cd $owd

uncompyle6/scanner.py

@@ -602,27 +602,6 @@ class Scanner:
         return self.Token
 
 
-# TODO: after the next xdis release, use from there instead.
-def parse_fn_counts_30_35(argc):
-    """
-    In Python 3.0 to 3.5 MAKE_CLOSURE and MAKE_FUNCTION encode
-    arguments counts of positional, default + named, and annotation
-    arguments a particular kind of encoding where each of
-    the entry a a packed byted value of the lower 24 bits
-    of ``argc``.  The high bits of argc may have come from
-    an EXTENDED_ARG instruction. Here, we unpack the values
-    from the ``argc`` int and return a triple of the
-    positional args, named_args, and annotation args.
-    """
-    annotate_count = (argc >> 16) & 0x7FFF
-    # For some reason that I don't understand, annotate_args is off by one
-    # when there is an EXENDED_ARG instruction from what is documented in
-    # https://docs.python.org/3.4/library/dis.html#opcode-MAKE_CLOSURE
-    if annotate_count > 1:
-        annotate_count -= 1
-    return ((argc & 0xFF), (argc >> 8) & 0xFF, annotate_count)
-
-
 def get_scanner(version, is_pypy=False, show_asm=None):
     # If version is a string, turn that into the corresponding float.
     if isinstance(version, str):
@@ -677,6 +656,16 @@ def get_scanner(version, is_pypy=False, show_asm=None):
     return scanner
 
 
+def prefer_double_quote(string: str) -> str:
+    """
+    Prefer a double quoted string over a
+    single quoted string when possible
+    """
+    if string.find("'") == -1:
+        return f'"{string}"'
+    return repr(string)
+
+
 if __name__ == "__main__":
     import inspect
 

uncompyle6/scanners/scanner3.py

@@ -43,8 +43,9 @@ import xdis
 import xdis.opcodes.opcode_33 as op3
 from xdis import Instruction, instruction_size, iscode
 from xdis.bytecode import _get_const_info
+from xdis.opcodes.opcode_3x import parse_fn_counts_30_35
 
-from uncompyle6.scanner import CONST_COLLECTIONS, Scanner, parse_fn_counts_30_35
+from uncompyle6.scanner import CONST_COLLECTIONS, Scanner, prefer_double_quote
 from uncompyle6.scanners.tok import Token
 from uncompyle6.util import get_code_name
 
@@ -608,6 +609,7 @@ class Scanner3(Scanner):
                     pattr = "<code_object " + co_name + ">"
                 elif isinstance(const, str):
                     opname = "LOAD_STR"
+                    pattr = prefer_double_quote(inst.argval)
                 else:
                     if isinstance(inst.arg, int) and inst.arg < len(co.co_consts):
                         argval, _ = _get_const_info(inst.arg, co.co_consts)

uncompyle6/scanners/scanner37base.py

@@ -38,7 +38,7 @@ import xdis.opcodes.opcode_37 as op3
 from xdis import Instruction, instruction_size, iscode
 from xdis.bytecode import _get_const_info
 
-from uncompyle6.scanner import Scanner, Token
+from uncompyle6.scanner import Scanner, Token, prefer_double_quote
 
 globals().update(op3.opmap)
 
@@ -383,6 +383,7 @@ class Scanner37Base(Scanner):
                     pattr = "<code_object " + const.co_name + ">"
                 elif isinstance(const, str):
                     opname = "LOAD_STR"
+                    pattr = prefer_double_quote(inst.argval)
                 else:
                     if isinstance(inst.arg, int) and inst.arg < len(co.co_consts):
                         argval, _ = _get_const_info(inst.arg, co.co_consts)

uncompyle6/semantics/consts.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2023 by Rocky Bernstein
+#  Copyright (c) 2017-2024 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -14,9 +14,11 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Constants and initial table values used in pysource.py and fragments.py"""
 
-import re, sys
+import re
+import sys
+
 from uncompyle6.parsers.treenode import SyntaxTree
-from uncompyle6.scanners.tok import Token, NoneToken
+from uncompyle6.scanners.tok import NoneToken, Token
 
 minint = -sys.maxsize - 1
 maxint = sys.maxsize
@@ -46,6 +48,7 @@ maxint = sys.maxsize
 # call((.. op ..)).
 
 NO_PARENTHESIS_EVER = 100
+PARENTHESIS_ALWAYS = -2
 
 # fmt: off
 PRECEDENCE = {

uncompyle6/semantics/n_actions.py

@@ -18,7 +18,14 @@ Custom Nonterminal action functions. See NonterminalActions docstring.
 
 from uncompyle6.parsers.treenode import SyntaxTree
 from uncompyle6.scanners.tok import Token
-from uncompyle6.semantics.consts import INDENT_PER_LEVEL, NONE, PRECEDENCE, minint
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL,
+    NO_PARENTHESIS_EVER,
+    NONE,
+    PARENTHESIS_ALWAYS,
+    PRECEDENCE,
+    minint,
+)
 from uncompyle6.semantics.helper import find_code_node, flatten_list
 from uncompyle6.util import better_repr, get_code_name
 
@@ -39,8 +46,9 @@ class NonterminalActions:
         # parenthesis surrounding it. A high value indicates no
         # parenthesis are needed.
         self.prec = 1000
+        self.in_format_string = False
 
-    def n_alias(self, node):
+    def n_alias(self, node: SyntaxTree):
         if self.version <= (2, 1):
             if len(node) == 2:
                 store = node[1]
@@ -65,7 +73,7 @@ class NonterminalActions:
 
     n_alias37 = n_alias
 
-    def n_assign(self, node):
+    def n_assign(self, node: SyntaxTree):
         # A horrible hack for Python 3.0 .. 3.2
         if (3, 0) <= self.version <= (3, 2) and len(node) == 2:
             if (
@@ -76,19 +84,19 @@ class NonterminalActions:
                 self.prune()
         self.default(node)
 
-    def n_assign2(self, node):
+    def n_assign2(self, node: SyntaxTree):
         for n in node[-2:]:
             if n[0] == "unpack":
                 n[0].kind = "unpack_w_parens"
         self.default(node)
 
-    def n_assign3(self, node):
+    def n_assign3(self, node: SyntaxTree):
         for n in node[-3:]:
             if n[0] == "unpack":
                 n[0].kind = "unpack_w_parens"
         self.default(node)
 
-    def n_attribute(self, node):
+    def n_attribute(self, node: SyntaxTree):
         if node[0] == "LOAD_CONST" or node[0] == "expr" and node[0][0] == "LOAD_CONST":
             # FIXME: I didn't record which constants parenthesis is
             # necessary. However, I suspect that we could further
@@ -98,7 +106,7 @@ class NonterminalActions:
             node.kind = "attribute_w_parens"
         self.default(node)
 
-    def n_bin_op(self, node):
+    def n_bin_op(self, node: SyntaxTree):
         """bin_op (formerly "binary_expr") is the Python AST BinOp"""
         self.preorder(node[0])
         self.write(" ")
@@ -110,9 +118,9 @@ class NonterminalActions:
         self.prec += 1
         self.prune()
 
-    def n_build_slice2(self, node):
+    def n_build_slice2(self, node: SyntaxTree):
         p = self.prec
-        self.prec = 100
+        self.prec = NO_PARENTHESIS_EVER
         if not node[0].isNone():
             self.preorder(node[0])
         self.write(":")
@@ -121,9 +129,9 @@ class NonterminalActions:
         self.prec = p
         self.prune()  # stop recursing
 
-    def n_build_slice3(self, node):
+    def n_build_slice3(self, node: SyntaxTree):
         p = self.prec
-        self.prec = 100
+        self.prec = NO_PARENTHESIS_EVER
         if not node[0].isNone():
             self.preorder(node[0])
         self.write(":")
@@ -135,7 +143,7 @@ class NonterminalActions:
         self.prec = p
         self.prune()  # stop recursing
 
-    def n_classdef(self, node):
+    def n_classdef(self, node: SyntaxTree):
         if self.version >= (3, 6):
             self.n_classdef36(node)
         elif self.version >= (3, 0):
@@ -198,7 +206,7 @@ class NonterminalActions:
 
     n_classdefdeco2 = n_classdef
 
-    def n_const_list(self, node):
+    def n_const_list(self, node: SyntaxTree):
         """
         prettyprint a constant dict, list, set or tuple.
         """
@@ -292,7 +300,7 @@ class NonterminalActions:
         self.prune()
         return
 
-    def n_delete_subscript(self, node):
+    def n_delete_subscript(self, node: SyntaxTree):
         if node[-2][0] == "build_list" and node[-2][0][-1].kind.startswith(
             "BUILD_TUPLE"
         ):
@@ -302,7 +310,7 @@ class NonterminalActions:
 
     n_store_subscript = n_subscript = n_delete_subscript
 
-    def n_dict(self, node):
+    def n_dict(self, node: SyntaxTree):
         """
         Prettyprint a dict.
         'dict' is something like k = {'a': 1, 'b': 42}"
@@ -314,7 +322,7 @@ class NonterminalActions:
             return
 
         p = self.prec
-        self.prec = 100
+        self.prec = PRECEDENCE["dict"]
 
         self.indent_more(INDENT_PER_LEVEL)
         sep = INDENT_PER_LEVEL[:-1]
@@ -326,8 +334,8 @@ class NonterminalActions:
             if node[0].kind.startswith("kvlist"):
                 # Python 3.5+ style key/value list in dict
                 kv_node = node[0]
-                l = list(kv_node)
+                ll = list(kv_node)
-                length = len(l)
+                length = len(ll)
                 if kv_node[-1].kind.startswith("BUILD_MAP"):
                     length -= 1
                 i = 0
@@ -335,7 +343,7 @@ class NonterminalActions:
                 # Respect line breaks from source
                 while i < length:
                     self.write(sep)
-                    name = self.traverse(l[i], indent="")
+                    name = self.traverse(ll[i], indent="")
                     if i > 0:
                         line_number = self.indent_if_source_nl(
                             line_number, self.indent + INDENT_PER_LEVEL[:-1]
@@ -343,7 +351,7 @@ class NonterminalActions:
                     line_number = self.line_number
                     self.write(name, ": ")
                     value = self.traverse(
-                        l[i + 1], indent=self.indent + (len(name) + 2) * " "
+                        ll[i + 1], indent=self.indent + (len(name) + 2) * " "
                     )
                     self.write(value)
                     sep = ", "
@@ -356,15 +364,15 @@ class NonterminalActions:
             elif len(node) > 1 and node[1].kind.startswith("kvlist"):
                 # Python 3.0..3.4 style key/value list in dict
                 kv_node = node[1]
-                l = list(kv_node)
+                ll = list(kv_node)
-                if len(l) > 0 and l[0].kind == "kv3":
+                if len(ll) > 0 and ll[0].kind == "kv3":
                     # Python 3.2 does this
                     kv_node = node[1][0]
-                    l = list(kv_node)
+                    ll = list(kv_node)
                 i = 0
-                while i < len(l):
+                while i < len(ll):
                     self.write(sep)
-                    name = self.traverse(l[i + 1], indent="")
+                    name = self.traverse(ll[i + 1], indent="")
                     if i > 0:
                         line_number = self.indent_if_source_nl(
                             line_number, self.indent + INDENT_PER_LEVEL[:-1]
@@ -373,7 +381,7 @@ class NonterminalActions:
                     line_number = self.line_number
                     self.write(name, ": ")
                     value = self.traverse(
-                        l[i], indent=self.indent + (len(name) + 2) * " "
+                        ll[i], indent=self.indent + (len(name) + 2) * " "
                     )
                     self.write(value)
                     sep = ", "
@@ -591,7 +599,7 @@ class NonterminalActions:
             self.println(lines[-1], quote)
         self.prune()
 
-    def n_elifelsestmtr(self, node):
+    def n_elifelsestmtr(self, node: SyntaxTree):
         if node[2] == "COME_FROM":
             return_stmts_node = node[3]
             node.kind = "elifelsestmtr2"
@@ -622,7 +630,7 @@ class NonterminalActions:
         self.indent_less()
         self.prune()
 
-    def n_except_cond2(self, node):
+    def n_except_cond2(self, node: SyntaxTree):
         if node[-1] == "come_from_opt":
             unpack_node = -3
         else:
@@ -636,7 +644,7 @@ class NonterminalActions:
     # FIXME: figure out how to get this into customization
     # put so that we can get access via super from
     # the fragments routine.
-    def n_exec_stmt(self, node):
+    def n_exec_stmt(self, node: SyntaxTree):
         """
         exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
         exec_stmt ::= expr exprlist EXEC_STMT
@@ -668,7 +676,9 @@ class NonterminalActions:
         #     hasattr(self, 'current_line_number')):
         #     self.source_linemap[self.current_line_number] = n.linestart
 
-        self.prec = PRECEDENCE.get(n.kind, -2)
+        if n.kind != "expr":
+            self.prec = PRECEDENCE.get(n.kind, PARENTHESIS_ALWAYS)
+
         if n == "LOAD_CONST" and repr(n.pattr)[0] == "-":
             self.prec = 6
 
@@ -725,7 +735,7 @@ class NonterminalActions:
         self.write(")")
         self.prune()
 
-    n_generator_exp_async = n_generator_exp
+    n_genexpr_func = n_generator_exp_async = n_generator_exp
 
     def n_ifelsestmtr(self, node):
         if node[2] == "COME_FROM":
@@ -805,7 +815,7 @@ class NonterminalActions:
         self.make_function(node, is_lambda=True, code_node=node[-2])
         self.prune()  # stop recursing
 
-    def n_list(self, node):
+    def n_list(self, node: SyntaxTree):
         """
         prettyprint a dict, list, set or tuple.
         """
@@ -836,13 +846,16 @@ class NonterminalActions:
         if lastnodetype.startswith("BUILD_LIST"):
             self.write("[")
             endchar = "]"
+
         elif lastnodetype.startswith("BUILD_MAP_UNPACK"):
             self.write("{*")
             endchar = "}"
+
         elif lastnodetype.startswith("BUILD_SET"):
             self.write("{")
             endchar = "}"
-        elif lastnodetype.startswith("BUILD_TUPLE"):
+
+        elif lastnodetype.startswith("BUILD_TUPLE") or node == "tuple":
             # Tuples can appear places that can NOT
             # have parenthesis around them, like array
             # subscripts. We check for that by seeing
@@ -863,6 +876,7 @@ class NonterminalActions:
         elif lastnodetype.startswith("ROT_TWO"):
             self.write("(")
             endchar = ")"
+
         else:
             raise TypeError(
                 "Internal Error: n_build_list expects list, tuple, set, or unpack"
@@ -901,7 +915,7 @@ class NonterminalActions:
         self.prune()
         return
 
-    n_set = n_tuple = n_build_set = n_list
+    n_set = n_build_set = n_tuple = n_list
 
     def n_list_comp(self, node):
         """List comprehensions"""