Merge branch 'master' into python-2.4

.github/ISSUE_TEMPLATE/bug-report.md

@@ -4,25 +4,40 @@ about: Tell us about uncompyle6 bugs
 
 ---
 
-<!-- __Note:__ Bugs are not for asking questions about a problem you are trying to solve that involve the use of uncompyle6 along the way, although I may be more tolerent of this if you sponsor the project. Also, the unless you are a sponsor of the project, it may take a while, maybe a week or so, before the bug report is noticed, let alone acted upon. To set expectations, some legitimate bugs can take years to fix. But they eventually do get fixed. Funding the project was added to address the problem that there are lots of people seeking help and reporting bugs, but few people who are willing to provide support on the other side. Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ?
+<!-- __Note:__ Bugs are not for asking questions about a problem you
+are trying to solve that involve the use of uncompyle6 along the way,
+although I may be more tolerent of this if you sponsor the project.
+
+Also, the unless you are a sponsor of the project, it may take a
+while, maybe a week or so, before the bug report is noticed, let alone
+acted upon.
+
+To set expectations, some legitimate bugs can take years
+to fix, but they eventually do get fixed. Funding the project was
+added to address the problem that there are lots of people seeking
+help and reporting bugs, but few people who are willing or capable of
+providing help or fixing bugs.
+
+Finally, have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md
+?
 
 
 Please remove any of the optional sections if they are not applicable.
 
-Prerequisites
+Prerequisites/Caveats
 
 * Make sure the bytecode you have can be disassembled with a
   disassembler and produces valid results.
 * Don't put bytecode and corresponding source code on any service that
   requires registration to download.
-* When you open a bug report there is no privacy. If you need privacy then
-  contact me by email and explain who you are and why you need privacy.
-  But you may mindful that you may be asked to sponsor the project for the
-  personal and private help that you are seeking.
-* If the legitimacy ofthe activity is deemed suspicous, I may flag it as suspicious,
+* When you open a bug report there is no privacy. If you need privacy, then
+  contact me by email and explain who you are and the need for privacy.
+  But be mindful that you may be asked to sponsor the project for the
+  personal and private help that you are requesting.
+* If the legitimacy of the activity is deemed suspicous, I may flag it as suspicious,
   making the issue even more easy to detect.
 
-Bug reports that violate a prerequisite may be discarded.
+Bug reports that violate the above may be discarded.
 
 -->
 

README.rst

@@ -215,7 +215,8 @@ valid bytecode before trying this tool. This program can't decompile
 Microsoft Windows EXE files created by Py2EXE_, although we can
 probably decompile the code after you extract the bytecode
 properly. Handling pathologically long lists of expressions or
-statements is slow. We don't handle Cython_ or MicroPython_ which don't use bytecode.
+statements is slow. We don't handle Cython_ or MicroPython which don't
+use bytecode.
 
 There are numerous bugs in decompilation. And that's true for every
 other CPython decompiler I have encountered, even the ones that
@@ -262,7 +263,6 @@ See Also
 
 
 .. _Cython: https://en.wikipedia.org/wiki/Cython
-.. _MicroPython: https://micropotyon.org
 .. _trepan: https://pypi.python.org/pypi/trepan2g
 .. _compiler: https://pypi.python.org/pypi/spark_parser
 .. _HISTORY: https://github.com/rocky/python-uncompyle6/blob/master/HISTORY.md

test/add-test.py

@@ -3,15 +3,22 @@
 """
 import os, sys, py_compile
 
-assert len(sys.argv) >= 2
+assert (2 <= len(sys.argv) <= 4)
 version = sys.version[0:3]
 vers = sys.version_info[:2]
 if sys.argv[1] in ("--run", "-r"):
     suffix = "_run"
     py_source = sys.argv[2:]
+    i = 2
 else:
     suffix = ""
     py_source = sys.argv[1:]
+    i = 1
+try:
+    optimize = int(sys.argv[-1])
+    py_source = sys.argv[i:-1]
+except:
+    optimize = 2
 
 for path in py_source:
     short = os.path.basename(path)
@@ -20,8 +27,8 @@ for path in py_source:
     else:
         cfile = "bytecode_%s%s/%s" % (version, suffix, short) + "c"
     print("byte-compiling %s to %s" % (path, cfile))
-    optimize = 2
-    if vers >= (3, 0):
+    optimize = optimize
+    if vers > (3, 1):
         py_compile.compile(path, cfile, optimize=optimize)
     else:
         py_compile.compile(path, cfile)

test/simple_source/bug27+/01_module_doc.py

@@ -0,0 +1,9 @@
+# From 2.7.17 test_bdb.py
+# The problem was detecting a docstring at the begining of the module
+# It must be detected and change'd or else the "from __future__" below
+# is invalid.
+# Note that this has to be compiled with optimation < 2 or else optimization
+# will remove the docstring
+"""Rational, infinite-precision, real numbers."""
+
+from __future__ import division

test/simple_source/bug27+/03_doc_assign.py

@@ -0,0 +1,27 @@
+# From 2.7 test_descr.py
+# Testing __doc__ descriptor...
+# The bug in decompilation was erroneously matching
+# __doc__ = as a docstring
+
+"""This program is self-checking!"""
+
+def test_doc_descriptor():
+    # Testing __doc__ descriptor...
+    # Python SF bug 542984
+    class DocDescr(object):
+        def __get__(self, object, otype):
+            if object:
+                object = object.__class__.__name__ + ' instance'
+            if otype:
+                otype = otype.__name__
+            return 'object=%s; type=%s' % (object, otype)
+    class OldClass:
+        __doc__ = DocDescr()
+    class NewClass(object):
+        __doc__ = DocDescr()
+    assert OldClass.__doc__ == 'object=None; type=OldClass'
+    assert OldClass().__doc__ == 'object=OldClass instance; type=OldClass'
+    assert NewClass.__doc__ == 'object=None; type=NewClass'
+    assert NewClass().__doc__ == 'object=NewClass instance; type=NewClass'
+
+test_doc_descriptor()

uncompyle6/main.py

@@ -106,7 +106,9 @@ def decompile(
           )
     )
     if bytecode_version >= 3.0:
-        write("# Warning: this version has problems handling the Python 3 byte type in constants properly.\n")
+        write(
+            "# Warning: this version of Python has problems handling the Python 3 byte type in constants properly.\n"
+        )
     if co.co_filename:
         write("# Embedded file name: %s" % co.co_filename,)
     if timestamp:

uncompyle6/parser.py

@@ -621,7 +621,7 @@ class PythonParser(GenericASTBuilder):
         """
 
 
-def parse(p, tokens, customize):
+def parse(p, tokens, customize, code):
     p.customize_grammar_rules(tokens, customize)
     ast = p.parse(tokens)
     #  p.cleanup()
@@ -876,7 +876,7 @@ def python_parser(
     # parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
     #                 'showstack': 'full'}
     p = get_python_parser(version, parser_debug)
-    return parse(p, tokens, customize)
+    return parse(p, tokens, customize, co)
 
 
 if __name__ == "__main__":

uncompyle6/semantics/aligner.py

@@ -142,7 +142,7 @@ def code_deparse_align(co, out=sys.stderr, version=None, is_pypy=None,
                             is_pypy = is_pypy)
 
     isTopLevel = co.co_name == '<module>'
-    deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel)
+    deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel)
 
     assert deparsed.ast == 'stmts', 'Should have parsed grammar start'
 

uncompyle6/semantics/consts.py

@@ -142,17 +142,12 @@ PASS = SyntaxTree(
 )
 
 ASSIGN_DOC_STRING = lambda doc_string, doc_load: SyntaxTree(
-    "stmt",
+    "assign",
     [
         SyntaxTree(
-            "assign",
-            [
-                SyntaxTree(
-                    "expr", [Token(doc_load, pattr=doc_string, attr=doc_string)]
-                ),
-                SyntaxTree("store", [Token("STORE_NAME", pattr="__doc__")]),
-            ],
-        )
+            "expr", [Token(doc_load, pattr=doc_string, attr=doc_string)]
+        ),
+        SyntaxTree("store", [Token("STORE_NAME", pattr="__doc__")]),
     ],
 )
 

uncompyle6/semantics/customize3.py

@@ -90,7 +90,7 @@ def customize_for_version3(self, version):
         code_obj = node[1].attr
         assert iscode(code_obj)
         code = Code(code_obj, self.scanner, self.currentclass)
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
 
         # skip over: sstmt, stmt, return, ret_expr

uncompyle6/semantics/fragments.py

@@ -683,7 +683,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         assert iscode(cn.attr)
 
         code = Code(cn.attr, self.scanner, self.currentclass)
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
         ast = ast[0][0][0]
 
@@ -730,7 +730,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         code_name = code.co_name
         code = Code(code, self.scanner, self.currentclass)
 
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
 
         self.customize(code._customize)
         if ast[0] == "sstmt":
@@ -852,7 +852,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.prec = 27
 
         code = Code(node[1].attr, self.scanner, self.currentclass)
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
         if node == "set_comp":
             ast = ast[0][0][0]
@@ -997,7 +997,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.prec = 27
 
         code = Code(node[1].attr, self.scanner, self.currentclass)
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
         ast = ast[0][0][0]
         store = ast[3]
@@ -1153,9 +1153,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.name = old_name
         self.return_none = rn
 
-    def build_ast(
-        self, tokens, customize, is_lambda=False, noneInNames=False, isTopLevel=False
-    ):
+    def build_ast(self, tokens, customize, code, is_lambda=False,
+                  noneInNames=False, isTopLevel=False):
 
         # FIXME: DRY with pysource.py
 

uncompyle6/semantics/make_function2.py

@@ -96,6 +96,7 @@ def make_function2(self, node, is_lambda, nested=1, code_node=None):
         ast = self.build_ast(
             code._tokens,
             code._customize,
+            code,
             is_lambda=is_lambda,
             noneInNames=("None" in code.co_names),
         )

uncompyle6/semantics/make_function3.py

@@ -126,6 +126,7 @@ def make_function3_annotate(
         ast = self.build_ast(
             code._tokens,
             code._customize,
+            code,
             is_lambda=is_lambda,
             noneInNames=("None" in code.co_names),
         )
@@ -491,11 +492,14 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     defparams.reverse()
 
     try:
-        ast = self.build_ast(scanner_code._tokens,
-                             scanner_code._customize,
-                             is_lambda = is_lambda,
-                             noneInNames = ('None' in code.co_names))
-    except ParserError, p:
+        ast = self.build_ast(
+            scanner_code._tokens,
+            scanner_code._customize,
+            scanner_code,
+            is_lambda=is_lambda,
+            noneInNames=("None" in code.co_names),
+        )
+    except (ParserError(p), ParserError2(p)):
         self.write(str(p))
         if not self.tolerate_errors:
             self.ERROR = p

uncompyle6/semantics/make_function36.py

@@ -169,6 +169,7 @@ def make_function36(self, node, is_lambda, nested=1, code_node=None):
         ast = self.build_ast(
             scanner_code._tokens,
             scanner_code._customize,
+            scanner_code,
             is_lambda=is_lambda,
             noneInNames=("None" in code.co_names),
         )

uncompyle6/semantics/pysource.py

@@ -164,7 +164,6 @@ from uncompyle6.semantics.consts import (
     NONE,
     RETURN_NONE,
     PASS,
-    ASSIGN_DOC_STRING,
     NAME_MODULE,
     TAB,
     INDENT_PER_LEVEL,
@@ -1120,7 +1119,7 @@ class SourceWalker(GenericASTTraversal, object):
         assert iscode(cn.attr)
 
         code = Code(cn.attr, self.scanner, self.currentclass)
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
 
         # Remove single reductions as in ("stmts", "sstmt"):
@@ -1204,7 +1203,7 @@ class SourceWalker(GenericASTTraversal, object):
         assert iscode(code), node[code_index]
         code = Code(code, self.scanner, self.currentclass)
 
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
 
         # skip over: sstmt, stmt, return, ret_expr
@@ -1399,7 +1398,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec = 27
 
         code = Code(node[1].attr, self.scanner, self.currentclass)
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         self.customize(code._customize)
 
         # Remove single reductions as in ("stmts", "sstmt"):
@@ -2313,7 +2312,7 @@ class SourceWalker(GenericASTTraversal, object):
 
         indent = self.indent
         # self.println(indent, '#flags:\t', int(code.co_flags))
-        ast = self.build_ast(code._tokens, code._customize)
+        ast = self.build_ast(code._tokens, code._customize, code)
         code._tokens = None  # save memory
         assert ast == "stmts"
 
@@ -2324,6 +2323,7 @@ class SourceWalker(GenericASTTraversal, object):
         if ast[0] == "docstring":
             self.println(self.traverse(ast[0]))
             del ast[0]
+            first_stmt = ast[0]
 
         if 3.0 <= self.version <= 3.3:
             try:
@@ -2387,10 +2387,10 @@ class SourceWalker(GenericASTTraversal, object):
         # if docstring exists, dump it
         if code.co_consts and code.co_consts[0] is not None and len(ast) > 0:
             do_doc = False
-            if is_docstring(ast[0]):
+            if is_docstring(ast[0], self.version, code.co_consts):
                 i = 0
                 do_doc = True
-            elif len(ast) > 1 and is_docstring(ast[1]):
+            elif len(ast) > 1 and is_docstring(ast[1], self.version, code.co_consts):
                 i = 1
                 do_doc = True
             if do_doc and self.hide_internal:
@@ -2466,7 +2466,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.return_none = rn
 
     def build_ast(
-        self, tokens, customize, is_lambda=False, noneInNames=False, isTopLevel=False
+        self, tokens, customize, code, is_lambda=False, noneInNames=False, isTopLevel=False
     ):
 
         # FIXME: DRY with fragments.py
@@ -2486,14 +2486,14 @@ class SourceWalker(GenericASTTraversal, object):
                 p_insts = self.p.insts
                 self.p.insts = self.scanner.insts
                 self.p.offset2inst_index = self.scanner.offset2inst_index
-                ast = python_parser.parse(self.p, tokens, customize)
+                ast = python_parser.parse(self.p, tokens, customize, code)
                 self.customize(customize)
                 self.p.insts = p_insts
             except python_parser.ParserError, e:
                 raise ParserError(e, tokens, self.p.debug['reduce'])
             except AssertionError, e:
                 raise ParserError(e, tokens, self.p.debug['reduce'])
-            transform_ast = self.treeTransform.transform(ast)
+            transform_ast = self.treeTransform.transform(ast, code)
             self.maybe_show_tree(ast)
             del ast  # Save memory
             return transform_ast
@@ -2525,7 +2525,7 @@ class SourceWalker(GenericASTTraversal, object):
             self.p.insts = self.scanner.insts
             self.p.offset2inst_index = self.scanner.offset2inst_index
             self.p.opc = self.scanner.opc
-            ast = python_parser.parse(self.p, tokens, customize)
+            ast = python_parser.parse(self.p, tokens, customize, code)
             self.p.insts = p_insts
         except python_parser.ParserError, e:
             raise ParserError(e, tokens, self.p.debug['reduce'])
@@ -2533,7 +2533,7 @@ class SourceWalker(GenericASTTraversal, object):
         checker(ast, False, self.ast_errors)
 
         self.customize(customize)
-        transform_ast = self.treeTransform.transform(ast)
+        transform_ast = self.treeTransform.transform(ast, code)
 
         self.maybe_show_tree(ast)
 
@@ -2595,7 +2595,7 @@ def code_deparse(
     )
 
     isTopLevel = co.co_name == "<module>"
-    deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel)
+    deparsed.ast = deparsed.build_ast(tokens, customize, co, isTopLevel=isTopLevel)
 
     #### XXX workaround for profiling
     if deparsed.ast is None:

uncompyle6/semantics/transform.py

@@ -13,7 +13,6 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from xdis import iscode
 from uncompyle6.show import maybe_show_tree
 from copy import copy
 from spark_parser import GenericASTTraversal, GenericASTTraversalPruningException
@@ -21,16 +20,39 @@ from spark_parser import GenericASTTraversal, GenericASTTraversalPruningExceptio
 from uncompyle6.semantics.helper import find_code_node
 from uncompyle6.parsers.treenode import SyntaxTree
 from uncompyle6.scanners.tok import NoneToken, Token
-from uncompyle6.semantics.consts import RETURN_NONE
+from uncompyle6.semantics.consts import RETURN_NONE, ASSIGN_DOC_STRING
 
 
-def is_docstring(node):
+def is_docstring(node, version, co_consts):
     if node == "sstmt":
         node = node[0]
-    try:
-        return node.kind == "assign" and node[1][0].pattr == "__doc__"
-    except:
-        return False
+    # TODO: the test below on 2.7 succeeds for
+    #   class OldClass:
+    #     __doc__ = DocDescr()
+    # which produces:
+    #
+    #   assign (2)
+    #      0. expr
+    #         call (2)
+    #              0. expr
+    #                  L.  16         6  LOAD_DEREF            0  'DocDescr'
+    #              1.                 9  CALL_FUNCTION_0       0  None
+    #      1. store
+    #
+    # See Python 2.7 test_descr.py
+
+    # If ASSIGN_DOC_STRING doesn't work we need something like the below
+    # but more elaborate to address the above.
+
+    # try:
+    #     return node.kind == "assign" and node[1][0].pattr == "__doc__"
+    # except:
+    #     return False
+    if version <= 2.7:
+        doc_load = "LOAD_CONST"
+    else:
+        doc_load = "LOAD_STR"
+    return node == ASSIGN_DOC_STRING(co_consts[0], doc_load)
 
 
 def is_not_docstring(call_stmt_node):
@@ -417,7 +439,7 @@ class TreeTransform(GenericASTTraversal, object):
         node = self.preorder(node)
         return node
 
-    def transform(self, ast):
+    def transform(self, ast, code):
         self.maybe_show_tree(ast)
         self.ast = copy(ast)
         self.ast = self.traverse(self.ast, is_lambda=False)
@@ -438,9 +460,9 @@ class TreeTransform(GenericASTTraversal, object):
             for i in range(len(self.ast)):
                 sstmt = ast[i]
                 if len(sstmt) == 1 and sstmt == "sstmt":
-                    ast[i] = ast[i][0]
+                    self.ast[i] = self.ast[i][0]
 
-                if is_docstring(self.ast[i]):
+                if is_docstring(self.ast[i], self.version, code.co_consts):
                     load_const = self.ast[i].first_child()
                     docstring_ast = SyntaxTree(
                         "docstring",