Merge branch 'master' into python-2.4

test/simple_source/stmts/00_docstring.py

@@ -1,12 +1,55 @@
+# -*- coding: utf-8 -*-
 # uncompyle2 bug was not escaping """ properly
 
 # RUNNABLE!
 r'''func placeholder - with ("""\nstring\n""")'''
-def foo():
-    r'''func placeholder - ' and with ("""\nstring\n""")'''
 
-def bar():
+def dq0():
+    assert __doc__ == r'''func placeholder - with ("""\nstring\n""")'''
+
+def dq1():
+    """assert that dedent() has no effect on 'text'"""
+    assert dq1.__doc__ == """assert that dedent() has no effect on 'text'"""
+
+def dq2():
+    '''assert that dedent() has no effect on 'text\''''
+    assert dq1.__doc__ == '''assert that dedent() has no effect on 'text\''''
+
+def dq3():
+    """assert that dedent() has no effect on 'text\""""
+    assert dq3.__doc__ == """assert that dedent() has no effect on 'text\""""
+
+def dq4():
+    """assert that dedent() has no effect on 'text'"""
+    assert dq4.__doc__ == """assert that dedent() has no effect on 'text'"""
+
+def dq5():
+    r'''func placeholder - ' and with ("""\nstring\n""")'''
+    assert dq5.__doc__ == r'''func placeholder - ' and with ("""\nstring\n""")'''
+
+def dq6():
     r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+    assert dq6.__doc__ == r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
+
+def dq7():
+  u"""        <----- SEE 'u' HERE
+  >>> mylen(u"áéíóú")
+  5
+  """
+  assert dq7.__doc__ ==   u"""        <----- SEE 'u' HERE
+  >>> mylen(u"áéíóú")
+  5
+  """
+
+def dq8():
+  u"""        <----- SEE 'u' HERE
+  >>> mylen(u"تست")
+  5
+  """
+  assert dq8.__doc__ ==   u"""        <----- SEE 'u' HERE
+  >>> mylen(u"تست")
+  5
+  """
 
 def baz():
     """
@@ -22,9 +65,6 @@ def baz():
         >>> t.rundict(m1.__dict__, 'rundict_test_pvt')  # None are skipped.
         TestResults(failed=0, attempted=8)
     """
-    assert __doc__ == r'''func placeholder - with ("""\nstring\n""")'''
-    assert foo.__doc__ == r'''func placeholder - ' and with ("""\nstring\n""")'''
-    assert bar.__doc__ == r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
     assert baz.__doc__ == \
     """
         ...     '''>>> assert 1 == 1
@@ -40,4 +80,13 @@ def baz():
         TestResults(failed=0, attempted=8)
     """
 
+dq0()
+dq1()
+dq2()
+dq3()
+dq4()
+dq5()
+dq6()
+dq7()
+dq8()
 baz()

uncompyle6/bin/uncompile.py

@@ -40,6 +40,8 @@ Options:
   --weak-verify compile generated source
   --linemaps    generated line number correspondencies between byte-code
                 and generated source output
+  --encoding  <encoding>
+                use <encoding> in generated source according to pep-0263
   --help        show this message
 
 Debugging Options:
@@ -85,9 +87,10 @@ def main_bin():
                                     'timestamp tree tree+ '
                                     'fragments verify verify-run version '
                                     'weak-verify '
-                                    'showgrammar'.split(' '))
+                                    'showgrammar encoding='.split(' '))
     except getopt.GetoptError(e):
-        sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
+        sys.stderr.write('%s: %s\n' %
+                         (os.path.basename(sys.argv[0]), e))
         sys.exit(-1)
 
     options = {}
@@ -129,6 +132,8 @@ def main_bin():
             numproc = int(val)
         elif opt in ('--recurse', '-r'):
             recurse_dirs = True
+        elif opt == '--encoding':
+            options['source_encoding'] = val
         else:
             sys.stderr.write(opt)
             usage()

uncompyle6/main.py

@@ -42,7 +42,7 @@ def _get_outstream(outfile):
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
-        timestamp=None, showgrammar=False, code_objects={},
+        timestamp=None, showgrammar=False, source_encoding=None, code_objects={},
         source_size=None, is_pypy=None, magic_int=None,
         mapstream=None, do_fragments=False):
     """
@@ -81,6 +81,8 @@ def decompile(
         m = ""
 
     sys_version_lines = sys.version.split('\n')
+    if source_encoding:
+        write('# -*- coding: %s -*-' % source_encoding)
     write('# uncompyle6 version %s\n'
           '# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
           (VERSION, co_pypy_str, bytecode_version,
@@ -147,7 +149,7 @@ def compile_file(source_path):
 
 
 def decompile_file(filename, outstream=None, showasm=None, showast=False,
-                   showgrammar=False, mapstream=None, do_fragments=False):
+                   showgrammar=False, source_encoding=None, mapstream=None, do_fragments=False):
     """
     decompile Python byte-code file (.pyc). Return objects to
     all of the deparsed objects found in `filename`.
@@ -163,12 +165,12 @@ def decompile_file(filename, outstream=None, showasm=None, showast=False,
         for con in co:
             deparsed.append(
                 decompile(version, con, outstream, showasm, showast,
-                          timestamp, showgrammar, code_objects=code_objects,
+                          timestamp, showgrammar, source_encoding, code_objects=code_objects,
                           is_pypy=is_pypy, magic_int=magic_int),
                           mapstream=mapstream)
     else:
         deparsed = [decompile(version, co, outstream, showasm, showast,
-                              timestamp, showgrammar,
+                              timestamp, showgrammar, source_encoding,
                               code_objects=code_objects, source_size=source_size,
                               is_pypy=is_pypy, magic_int=magic_int,
                               mapstream=mapstream, do_fragments=do_fragments)]
@@ -179,7 +181,7 @@ def decompile_file(filename, outstream=None, showasm=None, showast=False,
 # FIXME: combine into an options parameter
 def main(in_base, out_base, compiled_files, source_files, outfile=None,
          showasm=None, showast=False, do_verify=False,
-         showgrammar=False, raise_on_error=False,
+         showgrammar=False, source_encoding=None, raise_on_error=False,
          do_linemaps=False, do_fragments=False):
     """
     in_base	base directory for input files
@@ -250,7 +252,7 @@ def main(in_base, out_base, compiled_files, source_files, outfile=None,
         # Try to uncompile the input file
         try:
             deparsed = decompile_file(infile, outstream, showasm, showast, showgrammar,
-                                      linemap_stream, do_fragments)
+                                      source_encoding, linemap_stream, do_fragments)
             if do_fragments:
                 for d in deparsed:
                     last_mod = None

uncompyle6/semantics/helper.py

@@ -100,13 +100,21 @@ def strip_quotes(str):
 
 def print_docstring(self, indent, docstring):
     quote = '"""'
-    if docstring.find("'''") == -1:
-        quote = "'''"
+    if docstring.find(quote) >= 0:
+        if docstring.find("'''") == -1:
+            quote = "'''"
+
     self.write(indent)
     if not PYTHON3 and not isinstance(docstring, str):
         # Must be unicode in Python2
         self.write('u')
         docstring = repr(docstring.expandtabs())[2:-1]
+    elif PYTHON3 and 2.4 <= self.version <= 2.7:
+        try:
+            repr(docstring.expandtabs())[1:-1].encode("ascii")
+        except UnicodeEncodeError:
+            self.write('u')
+        docstring = repr(docstring.expandtabs())[1:-1]
     else:
         docstring = repr(docstring.expandtabs())[1:-1]
 
@@ -130,10 +138,11 @@ def print_docstring(self, indent, docstring):
         # Restore backslashes unescaped since raw
         docstring = docstring.replace('\t', '\\')
     else:
-        # Escape '"' if it's the last character, so it doesn't
-        # ruin the ending triple quote
-        if len(docstring) and docstring[-1] == '"':
-            docstring = docstring[:-1] + '\\"'
+        # Escape the last character if it is the same as the
+        # triple quote character.
+        quote1 = quote[-1]
+        if len(docstring) and docstring[-1] == quote1:
+            docstring = docstring[:-1] + '\\' + quote1
 
         # Escape triple quote when needed
         if quote == '"""':
@@ -146,33 +155,22 @@ def print_docstring(self, indent, docstring):
         docstring = docstring.replace('\t', '\\\\')
 
     lines = docstring.split('\n')
-    calculate_indent = maxint
-    for line in lines[1:]:
-        stripped = line.lstrip()
-        if len(stripped) > 0:
-            calculate_indent = min(calculate_indent, len(line) - len(stripped))
-    calculate_indent = min(calculate_indent, len(lines[-1]) - len(lines[-1].lstrip()))
-    # Remove indentation (first line is special):
-
-    trimmed = [lines[0]]
-    if calculate_indent < maxint:
-        trimmed += [line[calculate_indent:] for line in lines[1:]]
 
     self.write(quote)
-    if len(trimmed) == 0:
+    if len(lines) == 0:
         self.println(quote)
-    elif len(trimmed) == 1:
-        self.println(trimmed[0], quote)
+    elif len(lines) == 1:
+        self.println(lines[0], quote)
     else:
-        self.println(trimmed[0])
-        for line in trimmed[1:-1]:
+        self.println(lines[0])
+        for line in lines[1:-1]:
             if line:
-                self.println( indent, line )
+                self.println( line )
             else:
                 self.println( "\n\n" )
                 pass
             pass
-        self.println(indent, trimmed[-1], quote)
+        self.println(lines[-1], quote)
     return True
 
 

uncompyle6/semantics/pysource.py

@@ -363,7 +363,10 @@ class SourceWalker(GenericASTTraversal, object):
     def write(self, *data):
         if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
             return
-        out = ''.join((str(j) for j in data))
+        if not PYTHON3:
+            out = ''.join((unicode(j) for j in data))
+        else:
+            out = ''.join((str(j) for j in data))
         n = 0
         for i in out:
             if i == '\n':
@@ -607,6 +610,11 @@ class SourceWalker(GenericASTTraversal, object):
             else:
                 self.write(repr(data))
         else:
+            if not PYTHON3:
+                try:
+                    repr(data).encode("ascii")
+                except UnicodeEncodeError:
+                    self.write('u')
             self.write(repr(data))
         # LOAD_CONST is a terminal, so stop processing/recursing early
         self.prune()