Revise format string handling

fstring_single{1,2} -> format_value{1,2} to match Python AST names
better

uncompyle6/parser.py

@@ -61,7 +61,6 @@ class PythonParser(GenericASTBuilder):
             'imports_cont',
             'kvlist_n',
             # Python 3.6+
-            'joined_str',
             'come_from_loops',
             ]
         self.collect = frozenset(nt_list)
@@ -83,7 +82,7 @@ class PythonParser(GenericASTBuilder):
         # FIXME: would love to do expr, sstmts, stmts and
         # so on but that would require major changes to the
         # semantic actions
-        self.singleton = frozenset(('str', 'joined_str', 'store', '_stmts', 'suite_stmts_opt',
+        self.singleton = frozenset(('str', 'store', '_stmts', 'suite_stmts_opt',
                                     'inplace_op'))
         # Instructions filled in from scanner
         self.insts = []

uncompyle6/parsers/parse36.py

@@ -188,21 +188,14 @@ class Python36Parser(Python35Parser):
                     self.add_unique_doc_rules(rules_str, customize)
             elif opname == 'FORMAT_VALUE':
                 rules_str = """
-                    expr            ::= fstring_single
+                    expr              ::= formatted_value1
-                    fstring_single  ::= expr FORMAT_VALUE
+                    formatted_value1  ::= expr FORMAT_VALUE
-                    expr            ::= fstring_expr
-                    fstring_expr    ::= expr FORMAT_VALUE
-
-                    str             ::= LOAD_CONST
-                    formatted_value ::= fstring_expr
-                    formatted_value ::= str
-
                 """
                 self.add_unique_doc_rules(rules_str, customize)
             elif opname == 'FORMAT_VALUE_ATTR':
                 rules_str = """
-                expr            ::= fstring_single
+                expr              ::= formatted_value2
-                fstring_single  ::= expr expr FORMAT_VALUE_ATTR
+                formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
                 """
                 self.add_unique_doc_rules(rules_str, customize)
             elif opname == 'MAKE_FUNCTION_8':
@@ -246,17 +239,12 @@ class Python36Parser(Python35Parser):
                 """
                 self.addRule(rules_str, nop_func)
 
-            elif opname == 'BUILD_STRING':
+            elif opname.startswith('BUILD_STRING'):
                 v = token.attr
-                joined_str_n = "formatted_value_%s" % v
                 rules_str = """
-                    expr                 ::= fstring_multi
+                    expr                 ::= joined_str
-                    fstring_multi        ::= joined_str BUILD_STRING
+                    joined_str           ::= %sBUILD_STRING_%d
-                    fstr                 ::= expr
+                """ % ("expr " * v, v)
-                    joined_str           ::= fstr+
-                    fstring_multi        ::= %s BUILD_STRING
-                    %s                   ::= %sBUILD_STRING
-                """ % (joined_str_n, joined_str_n, "formatted_value " * v)
                 self.add_unique_doc_rules(rules_str, customize)
                 if 'FORMAT_VALUE_ATTR' in self.seen_ops:
                     rules_str = """

uncompyle6/scanners/scanner36.py

@@ -33,6 +33,8 @@ class Scanner36(Scanner3):
                  t.op == self.opc.CALL_FUNCTION_EX and t.attr & 1):
                 t.kind = 'CALL_FUNCTION_EX_KW'
                 pass
+            elif t.op == self.opc.BUILD_STRING:
+                t.kind = 'BUILD_STRING_%s' % t.attr
             elif t.op == self.opc.CALL_FUNCTION_KW:
                 t.kind = 'CALL_FUNCTION_KW_%s' % t.attr
             elif t.op == self.opc.FORMAT_VALUE:

uncompyle6/semantics/customize36.py

@@ -45,13 +45,6 @@ def customize_for_version36(self, version):
     TABLE_DIRECT.update({
         'tryfinally36':     ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
                               (1, 'returns'), 3 ),
-        'fstring_expr':     ( "{%c%{conversion}}",
-                              (0, 'expr') ),
-        # FIXME: the below assumes the format strings
-        # don't have ''' in them. Fix this properly
-        'fstring_single':   ( "f'''{%c%{conversion}}'''", 0),
-        'formatted_value_attr': ( "f'''{%c%{conversion}}%{string}'''",
-                                  (0, 'expr')),
         'func_args36':      ( "%c(**", 0),
         'try_except36':     ( '%|try:\n%+%c%-%c\n\n', 1, -2 ),
         'except_return':    ( '%|except:\n%+%c%-', 3 ),
@@ -436,58 +429,49 @@ def customize_for_version36(self, version):
         else:
             data = fmt_node.attr
         node.conversion = FSTRING_CONVERSION_MAP.get(data, '')
+        return node.conversion
 
-    def n_fstring_expr(node):
+    def n_formatted_value1(node):
-        f_conversion(node)
+        expr = node[0]
-        self.default(node)
+        assert expr == 'expr'
-    self.n_fstring_expr = n_fstring_expr
+        value = self.traverse(expr, indent='')
-
+        conversion = f_conversion(node)
-    def n_fstr(node):
+        f_str = "f%s" % escape_string("{%s%s}" % (value, conversion))
-        if node[0] == 'expr' and node[0][0] == 'fstring_expr':
+        self.write(f_str)
-            f_conversion(node[0][0])
-            self.default(node[0][0])
-        else:
-            value = strip_quotes(self.traverse(node[0], indent=''))
-            pass
-        self.write(value)
         self.prune()
-    self.n_fstr = n_fstr
 
-    def n_fstring_single(node):
+    self.n_formatted_value1 = n_formatted_value1
-        attr4 = len(node) == 3 and node[-1] == 'FORMAT_VALUE_ATTR' and node[-1].attr == 4
+
-        if attr4 and hasattr(node[0][0], 'attr'):
+    def n_formatted_value2(node):
-            assert node[0] == 'expr'
+        expr = node[0]
+        assert expr == 'expr'
+        value = self.traverse(expr, indent='')
+        format_value_attr = node[-1]
+        assert format_value_attr == 'FORMAT_VALUE_ATTR'
+        attr = format_value_attr.attr
+        if attr == 4:
             assert node[1] == 'expr'
-            self.write("{%s:%s}" % (node[0][0].attr, node[1][0].attr))
+            fmt = strip_quotes(self.traverse(node[1], indent=''))
-            self.prune()
+            conversion = ":%s" % fmt
         else:
-            f_conversion(node)
+            conversion = FSTRING_CONVERSION_MAP.get(attr, '')
-            self.default(node)
+
-    self.n_fstring_single = n_fstring_single
+        f_str = "f%s" % escape_string("{%s%s}" % (value, conversion))
+        self.write(f_str)
+        self.prune()
+    self.n_formatted_value2 = n_formatted_value2
 
     def n_joined_str(node):
         result = ''
-        for fstr_node in node:
+        for expr in node[:-1]:
-            assert fstr_node == 'fstr'
+            assert expr == 'expr'
-            assert fstr_node[0] == 'expr'
+            value = self.traverse(expr, indent='')
-            subnode = fstr_node[0][0]
+            if expr[0].kind.startswith('formatted_value'):
-            if subnode.kind == 'fstring_expr':
+                # remove leading 'f'
-                # Don't include outer f'...'
+                value = value[1:]
-                f_conversion(subnode)
-                data = strip_quotes(self.traverse(subnode, indent=''))
-                result += data
-            elif subnode == 'LOAD_CONST':
-                result += strip_quotes(escape_string(subnode.attr))
-            elif subnode == 'fstring_single':
-                f_conversion(subnode)
-                data = self.traverse(subnode, indent='')
-                if data[0:1] == 'f':
-                    data = strip_quotes(data[1:])
-                result += data
-                pass
-            else:
-                result += strip_quotes(self.traverse(subnode, indent=''))
                 pass
+            # Remove leading quotes
+            result += strip_quotes(value)
             pass
         self.write('f%s' % escape_string(result))
         self.prune()

uncompyle6/semantics/pysource.py

@@ -1837,11 +1837,7 @@ class SourceWalker(GenericASTTraversal, object):
             typ = m.group('type') or '{'
             node = startnode
             if m.group('child'):
-                try:
+                node = node[int(m.group('child'))]
-                    node = node[int(m.group('child'))]
-                except:
-                    from trepan.api import debug; debug()
-                    pass
 
             if   typ == '%':	self.write('%')
             elif typ == '+':