Merge branch 'master' into python-3.3-to-3.5

test/simple_source/bug38/02_fstring_debug.py

@@ -0,0 +1,32 @@
+# Tests new "debug" format new in 3.8.
+# Much of this is adapted from 3.8 test/test_fstring.py
+# RUNNABLE!
+
+"""This program is self-checking!"""
+
+# fmt: off
+# We want to use "=" and ":=" *without* the surrounding space to test format spec and "=" detection
+f'{f"{3.1415=:.1f}":*^20}' == '*****3.1415=3.1*****'
+
+y = 2
+def f(x, width):
+    return f'x={x*y:{width}}'
+
+assert f('foo', 10) ==  'x=foofoo    '
+
+x = 'bar'
+assert f(10, 10), 'x=        20'
+
+x = 'A string'
+f"x={x!r}" == 'x=' + repr(x)
+
+pi = 'π'
+assert f'alpha α {pi=} ω omega', "alpha α pi='π' ω omega"
+
+x = 20
+# This isn't an assignment expression, it's 'x', with a format
+# spec of '=10'.
+assert f'{x:=10}' ==  '        20'
+
+assert f'{(x:=10)}' ==  '10'
+assert x ==  10

uncompyle6/parsers/parse27.py

@@ -8,6 +8,7 @@ from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse2 import Python2Parser
 from uncompyle6.parsers.reducecheck import (
     aug_assign1_check,
+    ifelsestmt,
     or_check,
     tryelsestmt,
     except_handler,
@@ -231,9 +232,9 @@ class Python27Parser(Python2Parser):
 
         # FIXME: Put more in this table
         self.reduce_check_table = {
-            # "ifelsestmt": ifelsestmt,
             "aug_assign1": aug_assign1_check,
             "except_handler": except_handler,
+            "ifelsestmt": ifelsestmt,
             "or": or_check,
             "tryelsestmt": tryelsestmt,
             "tryelsestmtl": tryelsestmt,

uncompyle6/parsers/parse38.py

@@ -20,6 +20,7 @@ from __future__ import print_function
 from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse37 import Python37Parser
+from uncompyle6.parsers.reducecheck import joined_str_check
 
 class Python38Parser(Python37Parser):
     def p_38_stmt(self, args):
@@ -521,6 +522,29 @@ class Python38Parser(Python37Parser):
                     self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
                     continue
                 continue
+
+            elif opname == "BUILD_STRING_2":
+               self.addRule(
+                   """
+                     expr                  ::= formatted_value_debug
+                     formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
+                     formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
+                   """,
+                   nop_func,
+               )
+               custom_ops_processed.add(opname)
+
+            elif opname == "BUILD_STRING_3":
+               self.addRule(
+                   """
+                     expr                  ::= formatted_value_debug
+                     formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3
+                     formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3
+                   """,
+                   nop_func,
+               )
+               custom_ops_processed.add(opname)
+
             elif opname == "LOAD_CLOSURE":
                 self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
 

uncompyle6/parsers/reducecheck/ifelsestmt.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2020-2021 Rocky Bernstein
+#  Copyright (c) 2020-2022 Rocky Bernstein
 
 from uncompyle6.scanners.tok import Token
 
@@ -125,7 +125,7 @@ IFELSE_STMT_RULES = frozenset(
         ),
     ])
 
-def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
+def ifelsestmt(self, lhs, n, rule, tree, tokens, first, last):
 
     if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP" and lhs != "ifelsestmtc":
         # ifelsestmt jumped outside of loop. No good.
@@ -142,7 +142,7 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
 
     # Avoid if/else where the "then" is a "raise_stmt1" for an
     # assert statement. Parse this as an "assert" instead.
-    stmts = ast[1]
+    stmts = tree[1]
     if stmts in ("c_stmts",) and len(stmts) == 1:
         raise_stmt1 = stmts[0]
         if (
@@ -156,8 +156,8 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
     # Since the come_froms are ordered so that lowest
     # offset COME_FROM is last, it is sufficient to test
     # just the last one.
-    if len(ast) == 5:
-        end_come_froms = ast[-1]
+    if len(tree) == 5:
+        end_come_froms = tree[-1]
         if end_come_froms.kind != "else_suite" and self.version >= (3, 0):
             if end_come_froms == "opt_come_from_except" and len(end_come_froms) > 0:
                 end_come_froms = end_come_froms[0]
@@ -170,21 +170,21 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
         # FIXME: There is weirdness in the grammar we need to work around.
         # we need to clean up the grammar.
         if self.version < (3, 0):
-            last_token = ast[-1]
+            last_token = tree[-1]
         else:
             last_token = tokens[last]
         if last_token == "COME_FROM" and tokens[first].offset > last_token.attr:
             if self.version < (3, 0) and self.insts[self.offset2inst_index[last_token.attr]].opname != "SETUP_LOOP":
                 return True
 
-    testexpr = ast[0]
+    testexpr = tree[0]
 
     # Check that the condition portion of the "if"
     # jumps to the "else" part.
     if testexpr[0] in ("testtrue", "testfalse"):
         if_condition = testexpr[0]
 
-        else_suite = ast[3]
+        else_suite = tree[3]
         assert else_suite.kind.startswith("else_suite")
 
         if len(if_condition) > 1 and if_condition[1].kind.startswith("jmp_"):
@@ -205,7 +205,7 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last):
 
             # FIXME: the below logic for jf_cfs could probably be
             # simplified.
-            jump_else_end = ast[2]
+            jump_else_end = tree[2]
             if jump_else_end == "jf_cf_pop":
                 jump_else_end = jump_else_end[0]
 

uncompyle6/parsers/reducecheck/iflaststmt.py

@@ -1,8 +1,22 @@
-#  Copyright (c) 2020 Rocky Bernstein
+#  Copyright (c) 2020, 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-def iflaststmt(self, lhs, n, rule, ast, tokens, first, last):
-    testexpr = ast[0]
+def iflaststmt(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    testexpr = tree[0]
 
     if testexpr[0] in ("testtrue", "testfalse"):
 

uncompyle6/parsers/reducecheck/joined_str_check.py

@@ -0,0 +1,47 @@
+#  Copyright (c) 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def joined_str_ok(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    # In Python 3.8, there is a new "=" specifier.
+    # See https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging
+    # We detect this here inside joined_str by looking for an
+    # expr->LOAD_STR which has an "=" added at the end
+    # and is equal without the "=" to expr->formated_value2->LOAD_CONST
+    # converted to a string.
+    expr1 = tree[0]
+    if expr1 != "expr":
+        return False
+    load_str = expr1[0]
+    if load_str != "LOAD_STR":
+        return False
+    format_value_equal = load_str.attr
+    if format_value_equal[-1] != "=":
+        return False
+    expr2 = tree[1]
+    if expr2 != "expr":
+        return False
+    formatted_value = expr2[0]
+    if not formatted_value.kind.startswith("formatted_value"):
+        return False
+    expr2a = formatted_value[0]
+    if expr2a != "expr":
+        return False
+    load_const = expr2a[0]
+    if load_const == "LOAD_CONST":
+        format_value2 = load_const.attr
+        return str(format_value2) == format_value_equal[:-1]
+    return True

uncompyle6/parsers/reducecheck/whilestmt.py

@@ -0,0 +1,31 @@
+#  Copyright (c) 2020 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def whilestmt(
+    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
+) -> bool:
+    # When we are missing a COME_FROM_LOOP, the
+    # "while" statement is nested inside an if/else
+    # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+    # Check this.
+    # print("XXX", first, last, rule)
+    # for t in range(first, last): print(tokens[t])
+    # print("="*40)
+
+    return tokens[last - 1] == "POP_BLOCK" and tokens[last] not in (
+        "JUMP_FORWARD",
+        "COME_FROM_LOOP",
+        "COME_FROM",
+    )

uncompyle6/parsers/reducecheck/whilestmt38.py

@@ -0,0 +1,41 @@
+#  Copyright (c) 2022 Rocky Bernstein
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+def whilestmt38_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # When we are missing a COME_FROM_LOOP, the
+    # "while" statement is nested inside an if/else
+    # so after the POP_BLOCK we have a JUMP_FORWARD which forms the "else" portion of the "if"
+    # Check this.
+    # print("XXX", first, last, rule)
+    # for t in range(first, last):
+    #     print(tokens[t])
+    # print("=" * 40)
+
+    if tokens[last] != "COME_FROM" and tokens[last - 1] == "COME_FROM":
+        last -= 1
+    if tokens[last - 1].kind.startswith("RAISE_VARARGS"):
+        return True
+    while tokens[last] == "COME_FROM":
+        last -= 1
+    # In a "while" loop, (in contrast to "for" loop), the loop jump is
+    # always to the first offset
+    first_offset = tokens[first].off2int()
+    if tokens[last] == "JUMP_LOOP" and (
+        tokens[last].attr == first_offset or tokens[last - 1].attr == first_offset
+    ):
+        return False
+    return True

uncompyle6/semantics/customize37.py

@@ -24,6 +24,8 @@ from uncompyle6.semantics.consts import (
 
 from uncompyle6.semantics.helper import flatten_list
 
+FSTRING_CONVERSION_MAP = {1: "!s", 2: "!r", 3: "!a", "X": ":X"}
+
 #######################
 def customize_for_version37(self, version):
     ########################
@@ -39,7 +41,9 @@ def customize_for_version37(self, version):
     PRECEDENCE["call_ex_kw4"]      =   1
     PRECEDENCE["call_kw"]          =   0
     PRECEDENCE["call_kw36"]        =   1
-    PRECEDENCE["formatted_value1"] = 100
+    PRECEDENCE["formatted_value1"] =  38 # f"...". This has to be below "named_expr" to make
+                                         # f'{(x := 10)}' preserve parenthesis
+    PRECEDENCE["formatted_value2"] =  38 # See above
     PRECEDENCE["if_exp_37a"]       =  28
     PRECEDENCE["if_exp_37b"]       =  28
     PRECEDENCE["dict_unpack"]      =   0  # **{...}

uncompyle6/semantics/customize38.py

@@ -20,6 +20,8 @@
 #######################
 
 from uncompyle6.semantics.consts import PRECEDENCE, TABLE_DIRECT
+from uncompyle6.semantics.customize37 import FSTRING_CONVERSION_MAP
+from uncompyle6.semantics.helper import escape_string, strip_quotes
 
 def customize_for_version38(self, version):
 
@@ -125,7 +127,7 @@ def customize_for_version38(self, version):
             "set_for": (" for %c in %c", (2, "store"), (0, "expr_or_arg"),),
             "whilestmt38": (
                 "%|while %c:\n%+%c%-\n\n",
-                (1, "testexpr"),
+                (1, ("bool_op", "testexpr", "testexprc")),
                 (2, ("l_stmts", "pass")),
             ),
             "whileTruestmt38": (
@@ -282,6 +284,54 @@ def customize_for_version38(self, version):
 
     self.n_set_afor = n_set_afor
 
+    def n_formatted_value_debug(node):
+        p = self.prec
+        self.prec = 100
+
+        formatted_value = node[1]
+        value_equal = node[0].attr
+        assert formatted_value.kind.startswith("formatted_value")
+        old_in_format_string = self.in_format_string
+        self.in_format_string = formatted_value.kind
+        format_value_attr = node[-1]
+
+        post_str = ""
+        if node[-1] == "BUILD_STRING_3":
+            post_load_str = node[-2]
+            post_str = self.traverse(post_load_str, indent="")
+            post_str = strip_quotes(post_str)
+
+        if format_value_attr == "FORMAT_VALUE_ATTR":
+            attr = format_value_attr.attr
+            if attr & 4:
+                fmt = strip_quotes(self.traverse(node[3], indent=""))
+                attr_flags = attr & 3
+                if attr_flags:
+                    conversion = "%s:%s" % (
+                        FSTRING_CONVERSION_MAP.get(attr_flags, ""),
+                        fmt,
+                    )
+                else:
+                    conversion = ":%s" % fmt
+            else:
+                conversion = FSTRING_CONVERSION_MAP.get(attr, "")
+            f_str = "f%s" % escape_string(
+                "{%s%s}%s" % (value_equal, conversion, post_str)
+            )
+        else:
+            f_conversion = self.traverse(formatted_value, indent="")
+            # Remove leaving "f" and quotes
+            conversion = strip_quotes(f_conversion[1:])
+            f_str = "f%s" % escape_string(f"{value_equal}{conversion}" + post_str)
+
+        self.write(f_str)
+        self.in_format_string = old_in_format_string
+
+        self.prec = p
+        self.prune()
+
+    self.n_formatted_value_debug = n_formatted_value_debug
+
     def n_suite_stmts_return(node):
         if len(node) > 1:
             assert len(node) == 2