Merge pull request #243 from rocky/docstrings-again

Some docstring bugs fixed, some remain...

test/simple_source/stmts/00_docstring.py

@@ -4,18 +4,52 @@
 # RUNNABLE!
 r'''func placeholder - with ("""\nstring\n""")'''
 
-def uni(word):
+def dq0():
+    assert __doc__ == r'''func placeholder - with ("""\nstring\n""")'''
+
+def dq1():
+    """assert that dedent() has no effect on 'text'"""
+    assert dq1.__doc__ == """assert that dedent() has no effect on 'text'"""
+
+def dq2():
+    '''assert that dedent() has no effect on 'text\''''
+    assert dq1.__doc__ == '''assert that dedent() has no effect on 'text\''''
+
+def dq3():
+    """assert that dedent() has no effect on 'text\""""
+    assert dq3.__doc__ == """assert that dedent() has no effect on 'text\""""
+
+def dq4():
+    """assert that dedent() has no effect on 'text'"""
+    assert dq4.__doc__ == """assert that dedent() has no effect on 'text'"""
+
+def dq5():
+    r'''func placeholder - ' and with ("""\nstring\n""")'''
+    assert dq5.__doc__ == r'''func placeholder - ' and with ("""\nstring\n""")'''
+
+def dq6():
+    r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+    assert dq6.__doc__ == r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+
+def dq7():
   u"""        <----- SEE 'u' HERE
   >>> mylen(u"áéíóú")
   5
   """
+  assert dq7.__doc__ ==   u"""        <----- SEE 'u' HERE
+  >>> mylen(u"áéíóú")
+  5
+  """
 
-
-def foo():
-    r'''func placeholder - ' and with ("""\nstring\n""")'''
-
-def bar():
-    r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+def dq8():
+  u"""        <----- SEE 'u' HERE
+  >>> mylen(u"تست")
+  5
+  """
+  assert dq8.__doc__ ==   u"""        <----- SEE 'u' HERE
+  >>> mylen(u"تست")
+  5
+  """
 
 def baz():
     """
@@ -31,9 +65,6 @@ def baz():
         >>> t.rundict(m1.__dict__, 'rundict_test_pvt')  # None are skipped.
         TestResults(failed=0, attempted=8)
     """
-    assert __doc__ == r'''func placeholder - with ("""\nstring\n""")'''
-    assert foo.__doc__ == r'''func placeholder - ' and with ("""\nstring\n""")'''
-    assert bar.__doc__ == r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
     assert baz.__doc__ == \
     """
         ...     '''>>> assert 1 == 1
@@ -48,9 +79,14 @@ def baz():
         >>> t.rundict(m1.__dict__, 'rundict_test_pvt')  # None are skipped.
         TestResults(failed=0, attempted=8)
     """
-    assert uni.__doc__ ==   u"""        <----- SEE 'u' HERE
-  >>> mylen(u"áéíóú")
-  5
-  """
 
+dq0()
+dq1()
+dq2()
+dq3()
+dq4()
+dq5()
+dq6()
+dq7()
+dq8()
 baz()

uncompyle6/semantics/helper.py

@@ -100,26 +100,21 @@ def strip_quotes(str):
 
 def print_docstring(self, indent, docstring):
     quote = '"""'
-    if docstring.find("'''") == -1:
-        quote = "'''"
+    if docstring.find(quote) >= 0:
+        if docstring.find("'''") == -1:
+            quote = "'''"
+
     self.write(indent)
     if not PYTHON3 and not isinstance(docstring, str):
         # Must be unicode in Python2
-        if self.version >= 2.4:
-            if self.version > 2.7:
-                docstring = repr(docstring.expandtabs())[2:-1].decode("unicode-escape")
-            else:
-                self.write('u')
-                docstring = repr(docstring.expandtabs())[2:-1].decode("string-escape").decode("utf-8")
-        else:
-            docstring = repr(docstring.expandtabs())[2:-1]
+        self.write('u')
+        docstring = repr(docstring.expandtabs())[2:-1]
     elif PYTHON3 and 2.4 <= self.version <= 2.7:
-        # TODO: check for unicode string
         try:
-            docstring = repr(docstring.expandtabs())[1:-1].encode("latin-1").decode("utf-8")
+            repr(docstring.expandtabs())[1:-1].encode("ascii")
         except UnicodeEncodeError:
             self.write('u')
-            docstring = repr(docstring.expandtabs())[1:-1]
+        docstring = repr(docstring.expandtabs())[1:-1]
     else:
         docstring = repr(docstring.expandtabs())[1:-1]
 
@@ -143,10 +138,11 @@ def print_docstring(self, indent, docstring):
         # Restore backslashes unescaped since raw
         docstring = docstring.replace('\t', '\\')
     else:
-        # Escape '"' if it's the last character, so it doesn't
-        # ruin the ending triple quote
-        if len(docstring) and docstring[-1] == '"':
-            docstring = docstring[:-1] + '\\"'
+        # Escape the last character if it is the same as the
+        # triple quote character.
+        quote1 = quote[-1]
+        if len(docstring) and docstring[-1] == quote1:
+            docstring = docstring[:-1] + '\\' + quote1
 
         # Escape triple quote when needed
         if quote == '"""':
@@ -159,33 +155,22 @@ def print_docstring(self, indent, docstring):
         docstring = docstring.replace('\t', '\\\\')
 
     lines = docstring.split('\n')
-    calculate_indent = maxint
-    for line in lines[1:]:
-        stripped = line.lstrip()
-        if len(stripped) > 0:
-            calculate_indent = min(calculate_indent, len(line) - len(stripped))
-    calculate_indent = min(calculate_indent, len(lines[-1]) - len(lines[-1].lstrip()))
-    # Remove indentation (first line is special):
-
-    trimmed = [lines[0]]
-    if calculate_indent < maxint:
-        trimmed += [line[calculate_indent:] for line in lines[1:]]
 
     self.write(quote)
-    if len(trimmed) == 0:
+    if len(lines) == 0:
         self.println(quote)
-    elif len(trimmed) == 1:
-        self.println(trimmed[0], quote)
+    elif len(lines) == 1:
+        self.println(lines[0], quote)
     else:
-        self.println(trimmed[0])
-        for line in trimmed[1:-1]:
+        self.println(lines[0])
+        for line in lines[1:-1]:
             if line:
-                self.println( indent, line )
+                self.println( line )
             else:
                 self.println( "\n\n" )
                 pass
             pass
-        self.println(indent, trimmed[-1], quote)
+        self.println(lines[-1], quote)
     return True
 
 

uncompyle6/semantics/pysource.py

@@ -610,6 +610,11 @@ class SourceWalker(GenericASTTraversal, object):
             else:
                 self.write(repr(data))
         else:
+            if not PYTHON3:
+                try:
+                    repr(data).encode("ascii")
+                except UnicodeEncodeError:
+                    self.write('u')
             self.write(repr(data))
         # LOAD_CONST is a terminal, so stop processing/recursing early
         self.prune()