Split out print_docstring

move from pysource.py to new helper.py

pytest/test_docstring.py

@@ -0,0 +1,78 @@
+import sys
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    from io import StringIO
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    from StringIO import StringIO
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+from uncompyle6.semantics.helper import print_docstring
+
+class PrintFake():
+    def __init__(self):
+        self.pending_newlines = 0
+        self.f = StringIO()
+
+    def write(self, *data):
+        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+            return
+        out = ''.join((str(j) for j in data))
+        n = 0
+        for i in out:
+            if i == '\n':
+                n += 1
+                if n == len(out):
+                    self.pending_newlines = max(self.pending_newlines, n)
+                    return
+            elif n:
+                self.pending_newlines = max(self.pending_newlines, n)
+                out = out[n:]
+                break
+            else:
+                break
+
+        if self.pending_newlines > 0:
+            self.f.write('\n'*self.pending_newlines)
+            self.pending_newlines = 0
+
+        for i in out[::-1]:
+            if i == '\n':
+                self.pending_newlines += 1
+            else:
+                break
+
+        if self.pending_newlines:
+            out = out[:-self.pending_newlines]
+        self.f.write(out)
+    def println(self, *data):
+        if data and not(len(data) == 1 and data[0] ==''):
+            self.write(*data)
+        self.pending_newlines = max(self.pending_newlines, 1)
+        return
+    pass
+
+def test_docstring():
+
+    for doc, expect in (
+            ("Now is the time",
+             '  """Now is the time"""'),
+           ("""
+Now is the time
+""",
+            '''  """
+  Now is the time
+  """''')
+
+            # (r'''func placeholder - ' and with ("""\nstring\n  """)''',
+            #  """  r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
+            # (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
+            # """  r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
+            ):
+
+        o = PrintFake()
+        # print(doc)
+        # print(expect)
+        print_docstring(o, '  ', doc)
+        assert expect == o.f.getvalue()

uncompyle6/linenumbers.py

@@ -1,4 +1,4 @@
-from collections import deque, namedtuple
+from collections import deque
 
 from xdis.code import iscode
 from xdis.load import load_file, load_module

uncompyle6/semantics/fragments.py

@@ -61,6 +61,7 @@ from uncompyle6.semantics import pysource
 from uncompyle6 import parser
 from uncompyle6.scanner import Token, Code, get_scanner
 from uncompyle6.semantics.check_ast import checker
+from uncompyle6.semantics.helper import print_docstring
 
 from uncompyle6.show import (
     maybe_show_asm,
@@ -1687,7 +1688,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
         if len(code.co_consts)>0 and code.co_consts[0] is not None and not isLambda: # ugly
             # docstring exists, dump it
-            self.print_docstring(indent, code.co_consts[0])
+            print_docstring(self, indent, code.co_consts[0])
 
         code._tokens = None # save memory
         assert ast == 'stmts'

uncompyle6/semantics/helper.py

@@ -0,0 +1,133 @@
+import sys
+from uncompyle6 import PYTHON3
+if PYTHON3:
+    minint = -sys.maxsize-1
+    maxint = sys.maxsize
+else:
+    minint = -sys.maxint-1
+    maxint = sys.maxint
+
+def print_docstring(self, indent, docstring):
+    ## FIXME: put this into a testable function.
+    if docstring.find('"""') == -1:
+        quote = '"""'
+    else:
+        quote = "'''"
+
+    self.write(indent)
+    if not PYTHON3 and not isinstance(docstring, str):
+        # Must be unicode in Python2
+        self.write('u')
+        docstring = repr(docstring.expandtabs())[2:-1]
+    else:
+        docstring = repr(docstring.expandtabs())[1:-1]
+
+    for (orig, replace) in (('\\\\', '\t'),
+                            ('\\r\\n', '\n'),
+                            ('\\n', '\n'),
+                            ('\\r', '\n'),
+                            ('\\"', '"'),
+                            ("\\'", "'")):
+        docstring = docstring.replace(orig, replace)
+
+    # Do a raw string if there are backslashes but no other escaped characters:
+    # also check some edge cases
+    if ('\t' in docstring
+        and '\\' not in docstring
+        and len(docstring) >= 2
+        and docstring[-1] != '\t'
+        and (docstring[-1] != '"'
+             or docstring[-2] == '\t')):
+        self.write('r') # raw string
+        # restore backslashes unescaped since raw
+        docstring = docstring.replace('\t', '\\')
+    else:
+        # Escape '"' if it's the last character, so it doesn't
+        # ruin the ending triple quote
+        if len(docstring) and docstring[-1] == '"':
+            docstring = docstring[:-1] + '\\"'
+        # Restore escaped backslashes
+        docstring = docstring.replace('\t', '\\\\')
+    # Escape triple quote when needed
+    if quote == '""""':
+        docstring = docstring.replace('"""', '\\"\\"\\"')
+    lines = docstring.split('\n')
+    calculate_indent = maxint
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if len(stripped) > 0:
+            calculate_indent = min(calculate_indent, len(line) - len(stripped))
+    calculate_indent = min(calculate_indent, len(lines[-1]) - len(lines[-1].lstrip()))
+    # Remove indentation (first line is special):
+    trimmed = [lines[0]]
+    if calculate_indent < maxint:
+        trimmed += [line[calculate_indent:] for line in lines[1:]]
+
+    self.write(quote)
+    if len(trimmed) == 0:
+        self.println(quote)
+    elif len(trimmed) == 1:
+        self.println(trimmed[0], quote)
+    else:
+        self.println(trimmed[0])
+        for line in trimmed[1:-1]:
+            self.println( indent, line )
+        self.println(indent, trimmed[-1], quote)
+
+# if __name__ == '__main__':
+#     if PYTHON3:
+#         from io import StringIO
+#     else:
+#         from StringIO import StringIO
+#     class PrintFake():
+#         def __init__(self):
+#             self.pending_newlines = 0
+#             self.f = StringIO()
+
+#         def write(self, *data):
+#             if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+#                 return
+#             out = ''.join((str(j) for j in data))
+#             n = 0
+#             for i in out:
+#                 if i == '\n':
+#                     n += 1
+#                     if n == len(out):
+#                         self.pending_newlines = max(self.pending_newlines, n)
+#                         return
+#                 elif n:
+#                     self.pending_newlines = max(self.pending_newlines, n)
+#                     out = out[n:]
+#                     break
+#                 else:
+#                     break
+
+#             if self.pending_newlines > 0:
+#                 self.f.write('\n'*self.pending_newlines)
+#                 self.pending_newlines = 0
+
+#             for i in out[::-1]:
+#                 if i == '\n':
+#                     self.pending_newlines += 1
+#                 else:
+#                     break
+
+#             if self.pending_newlines:
+#                 out = out[:-self.pending_newlines]
+#             self.f.write(out)
+#         def println(self, *data):
+#             if data and not(len(data) == 1 and data[0] ==''):
+#                 self.write(*data)
+#             self.pending_newlines = max(self.pending_newlines, 1)
+#             return
+#         pass
+
+#     for doc in (
+#             "Now is the time",
+#             r'''func placeholder - with ("""\nstring\n""")''',
+#             r'''func placeholder - ' and with ("""\nstring\n""")''',
+#             r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+#             ):
+#         o = PrintFake()
+#         print_docstring(o, '  ', doc)
+#         print(o.f.getvalue())

uncompyle6/semantics/make_function.py

@@ -8,6 +8,7 @@ from uncompyle6.scanner import Code
 from uncompyle6.parsers.astnode import AST
 from uncompyle6 import PYTHON3
 from uncompyle6.semantics.parser_error import ParserError
+from uncompyle6.semantics.helper import print_docstring
 
 if PYTHON3:
     from itertools import zip_longest
@@ -353,7 +354,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None):
 
     if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly
         # docstring exists, dump it
-        self.print_docstring(indent, code.co_consts[0])
+        print_docstring(self, indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'
@@ -542,7 +543,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
 
     if len(code.co_consts) > 0 and code.co_consts[0] is not None and not isLambda: # ugly
         # docstring exists, dump it
-        self.print_docstring(self.indent, code.co_consts[0])
+        print_docstring(self, self.indent, code.co_consts[0])
 
     code._tokens = None # save memory
     assert ast == 'stmts'

uncompyle6/semantics/pysource.py

@@ -83,6 +83,7 @@ from uncompyle6.semantics.make_function import (
     make_function2, make_function3, make_function3_annotate, find_globals)
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.check_ast import checker
+from uncompyle6.semantics.helper import print_docstring
 
 from uncompyle6.show import (
     maybe_show_ast,
@@ -740,73 +741,6 @@ class SourceWalker(GenericASTTraversal, object):
             self.write(*data)
         self.pending_newlines = max(self.pending_newlines, 1)
 
-    def print_docstring(self, indent, docstring):
-        ## FIXME: put this into a testable function.
-        if docstring.find('"""') == -1:
-            quote = '"""'
-        else:
-            quote = "'''"
-
-        self.write(indent)
-        if not PYTHON3 and not isinstance(docstring, str):
-            # Must be unicode in Python2
-            self.write('u')
-            docstring = repr(docstring.expandtabs())[2:-1]
-        else:
-            docstring = repr(docstring.expandtabs())[1:-1]
-
-        for (orig, replace) in (('\\\\', '\t'),
-                                ('\\r\\n', '\n'),
-                                ('\\n', '\n'),
-                                ('\\r', '\n'),
-                                ('\\"', '"'),
-                                ("\\'", "'")):
-            docstring = docstring.replace(orig, replace)
-
-        # Do a raw string if there are backslashes but no other escaped characters:
-        # also check some edge cases
-        if ('\t' in docstring
-            and '\\' not in docstring
-            and len(docstring) >= 2
-            and docstring[-1] != '\t'
-            and (docstring[-1] != '"'
-                 or docstring[-2] == '\t')):
-            self.write('r') # raw string
-            # restore backslashes unescaped since raw
-            docstring = docstring.replace('\t', '\\')
-        else:
-            # Escape '"' if it's the last character, so it doesn't
-            # ruin the ending triple quote
-            if len(docstring) and docstring[-1] == '"':
-                docstring = docstring[:-1] + '\\"'
-            # Restore escaped backslashes
-            docstring = docstring.replace('\t', '\\\\')
-        # Escape triple quote when needed
-        if quote == '""""':
-            docstring = docstring.replace('"""', '\\"\\"\\"')
-        lines = docstring.split('\n')
-        calculate_indent = maxint
-        for line in lines[1:]:
-            stripped = line.lstrip()
-            if len(stripped) > 0:
-                calculate_indent = min(calculate_indent, len(line) - len(stripped))
-        calculate_indent = min(calculate_indent, len(lines[-1]) - len(lines[-1].lstrip()))
-        # Remove indentation (first line is special):
-        trimmed = [lines[0]]
-        if calculate_indent < maxint:
-            trimmed += [line[calculate_indent:] for line in lines[1:]]
-
-        self.write(quote)
-        if len(trimmed) == 0:
-            self.println(quote)
-        elif len(trimmed) == 1:
-            self.println(trimmed[0], quote)
-        else:
-            self.println(trimmed[0])
-            for line in trimmed[1:-1]:
-                self.println( indent, line )
-            self.println(indent, trimmed[-1], quote)
-
     def is_return_none(self, node):
         # Is there a better way?
         ret = (node[0] == 'ret_expr'
@@ -2186,7 +2120,7 @@ class SourceWalker(GenericASTTraversal, object):
                     docstring = ast[i][0][0][0][0].pattr
                 except:
                     docstring = code.co_consts[0]
-                self.print_docstring(indent, docstring)
+                print_docstring(self, indent, docstring)
                 self.println()
                 del ast[i]
 
@@ -2313,7 +2247,7 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
     # convert leading '__doc__ = "..." into doc string
     try:
         if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]):
-            deparsed.print_docstring('', co.co_consts[0])
+            print_docstring(deparsed, '', co.co_consts[0])
             del deparsed.ast[0]
         if deparsed.ast[-1] == RETURN_NONE:
             deparsed.ast.pop() # remove last node