Merge branch 'master' into python-2.4

pytest/test_deparse.py

@@ -1,3 +1,4 @@
+import pytest
 from uncompyle6.semantics.fragments import deparse_code as deparse
 from uncompyle6 import PYTHON_VERSION, PYTHON3
 
@@ -32,19 +33,20 @@ def get_parsed_for_fn(fn):
     code = fn.func_code
     return deparse(PYTHON_VERSION, code)
 
-def check_expect(expect, parsed):
+def check_expect(expect, parsed, fn_name):
     debug = False
     i = 2
     max_expect = len(expect)
     for name, offset in sorted(parsed.offsets.keys()):
-        assert i+1 <= max_expect, "ran out if items in testing node"
+        assert i+1 <= max_expect, (
+            "%s: ran out if items in testing node" % fn_name)
         nodeInfo = parsed.offsets[name, offset]
         node = nodeInfo.node
         extractInfo = parsed.extract_node_info(node)
 
         assert expect[i] == extractInfo.selectedLine, \
-          ('line %s expect:\n%s\ngot:\n%s' %
+          ('%s: line %s expect:\n%s\ngot:\n%s' %
-                         (i, expect[i], extractInfo.selectedLine))
+               (fn_name, i, expect[i], extractInfo.selectedLine))
         assert expect[i+1] ==  extractInfo.markerLine, \
                      ('line %s expect:\n%s\ngot:\n%s' %
                      (i+1, expect[i+1], extractInfo.markerLine))
@@ -72,6 +74,7 @@ def check_expect(expect, parsed):
     pass
 
 
+@pytest.mark.skip(reason='needs reworking')
 def test_stuff():
     parsed = get_parsed_for_fn(map_stmts)
     expect = """
@@ -83,10 +86,10 @@ return (x, y)
 -------------
 0
 x = []
-    --
+    -
 Contained in...
 x = []
-------
+    --
 3
 x = []
 -
@@ -95,10 +98,10 @@ x = []
 ------
 6
 y = {}
-    --
+    -
 Contained in...
 y = {}
-------
+    --
 9
 y = {}
 -
@@ -130,7 +133,7 @@ Contained in...
 x = [] ...
 ------ ...
 """.split("\n")
-    check_expect(expect, parsed)
+    check_expect(expect, parsed, 'map_stmts')
     ########################################################
     # return
 
@@ -167,7 +170,7 @@ Contained in...
 return (x, y)
 -------------
 """.split("\n")
-    check_expect(expect, parsed)
+    check_expect(expect, parsed, 'return_stmt')
     ########################################################
 #     # try
 
@@ -315,4 +318,4 @@ for i in range(2): ...
 """.split("\n")
     parsed = get_parsed_for_fn(for_range_stmt)
     if not PYTHON3:
-        check_expect(expect, parsed)
+        check_expect(expect, parsed, 'range_stmt')

test/stdlib/runtests.sh

@@ -79,7 +79,7 @@ case $PYVERSION in
 	    [test_ioctl.py]=1 # Test takes too long to run
 	    [test_itertools.py]=1 # Syntax error - look at!
 	    [test_memoryio.py]=1 # FIX
-	    [test_multiprocessing.py]=1 # FIX
+	    [test_multiprocessing.py]=1 # On uncompyle2, taks 24 secs
 	    [test_pep352.py]=1 # ?
 	    [test_select.py]=1  # Runs okay but takes 11 seconds
 	    [test_socket.py]=1  # Runs ok but takes 22 seconds
@@ -105,7 +105,7 @@ cd $srcdir
 fulldir=$(pwd)
 
 # DECOMPILER=uncompyle2
-DECOMPILER="$fulldir/../../bin/uncompyle6"
+DECOMPILER=${DECOMPILER:-"$fulldir/../../bin/uncompyle6"}
 TESTDIR=/tmp/test${PYVERSION}
 if [[ -e $TESTDIR ]] ; then
     rm -fr $TESTDIR

uncompyle6/bin/uncompile.py

@@ -33,6 +33,7 @@ Options:
   -d            print timestamps
   -p <integer>  use <integer> number of processes
   -r            recurse directories looking for .pyc and .pyo files
+  --fragments   use fragments deparser
   --verify      compare generated source with input byte-code
   --verify-run  compile generated source, run it and check exit code
   --weak-verify compile generated source
@@ -84,9 +85,11 @@ def main_bin():
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
                                     'help asm grammar linemaps recurse timestamp tree '
-                                    'verify verify-run version showgrammar'.split(' '))
+                                    'fragments verify verify-run version weak-verify '
-    except getopt.GetoptError, e:
+                                    'showgrammar'.split(' '))
+    except getopt.GetoptError(e):
         sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
+        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
         sys.exit(-1)
 
     options = {}
@@ -101,6 +104,8 @@ def main_bin():
             options['do_verify'] = 'strong'
         elif opt == '--weak-verify':
             options['do_verify'] = 'weak'
+        elif opt == '--fragments':
+            options['do_fragments'] = True
         elif opt == '--verify-run':
             options['do_verify'] = 'verify-run'
         elif opt == '--linemaps':

uncompyle6/main.py

@@ -6,9 +6,10 @@ from uncompyle6.disas import check_object_path
 from uncompyle6.semantics import pysource
 from uncompyle6.parser import ParserError
 from uncompyle6.version import VERSION
-from uncompyle6.linenumbers import line_number_mapping
+# from uncompyle6.linenumbers import line_number_mapping
 
 from uncompyle6.semantics.pysource import deparse_code
+from uncompyle6.semantics.fragments import deparse_code as deparse_code_fragments
 from uncompyle6.semantics.linemap import deparse_code_with_map
 
 from xdis.load import load_module
@@ -28,7 +29,7 @@ def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,
         timestamp=None, showgrammar=False, code_objects={},
         source_size=None, is_pypy=False, magic_int=None,
-        mapstream=None):
+        mapstream=None, do_fragments=False):
     """
     ingests and deparses a given code block 'co'
 
@@ -89,9 +90,13 @@ def decompile(
                         sorted(deparsed.source_linemap.keys())]
             mapstream.write("\n\n# %s\n" % linemap)
         else:
-            deparsed = deparse_code(bytecode_version, co, out, showasm, showast,
+            if do_fragments:
-                                    showgrammar, code_objects=code_objects,
+                deparse_fn = deparse_code_fragments
-                                    is_pypy=is_pypy)
+            else:
+                deparse_fn = deparse_code
+            deparsed = deparse_fn(bytecode_version, co, out, showasm, showast,
+                                  showgrammar, code_objects=code_objects,
+                                      is_pypy=is_pypy)
             pass
         return deparsed
     except pysource.SourceWalkerError, e:
@@ -99,7 +104,7 @@ def decompile(
         raise pysource.SourceWalkerError(str(e))
 
 def decompile_file(filename, outstream=None, showasm=None, showast=False,
-                   showgrammar=False, mapstream=None):
+                   showgrammar=False, mapstream=None, do_fragments=False):
     """
     decompile Python byte-code file (.pyc). Return objects to
     all of the deparsed objects found in `filename`.
@@ -123,7 +128,7 @@ def decompile_file(filename, outstream=None, showasm=None, showast=False,
                               timestamp, showgrammar,
                               code_objects=code_objects, source_size=source_size,
                               is_pypy=is_pypy, magic_int=magic_int,
-                              mapstream=mapstream)]
+                              mapstream=mapstream, do_fragments=do_fragments)]
     co = None
     return deparsed
 
@@ -132,7 +137,7 @@ def decompile_file(filename, outstream=None, showasm=None, showast=False,
 def main(in_base, out_base, files, codes, outfile=None,
          showasm=None, showast=False, do_verify=False,
          showgrammar=False, raise_on_error=False,
-         do_linemaps=False):
+         do_linemaps=False, do_fragments=False):
     """
     in_base	base directory for input files
     out_base	base directory for output files (ignored when
@@ -196,7 +201,24 @@ def main(in_base, out_base, files, codes, outfile=None,
 
         # Try to uncompile the input file
         try:
-            decompile_file(infile, outstream, showasm, showast, showgrammar, linemap_stream)
+            deparsed = decompile_file(infile, outstream, showasm, showast, showgrammar,
+                                      linemap_stream, do_fragments)
+            if do_fragments:
+                for d in deparsed:
+                    last_mod = None
+                    offsets = d.offsets
+                    for e in sorted(offsets.keys()):
+                        if e[0] != last_mod:
+                            line = '=' * len(e[0])
+                            outstream.write("%s\n%s\n%s\n" % (line, e[0], line))
+                        last_mod = e[0]
+                        info = offsets[e]
+                        extractInfo = d.extract_node_info(info)
+                        outstream.write("%s" % info.node.format().strip() + "\n")
+                        outstream.write(extractInfo.selectedLine + "\n")
+                        outstream.write(extractInfo.markerLine + "\n\n")
+                    pass
+                pass
             tot_files += 1
         except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
             sys.stdout.write("\n")

uncompyle6/scanner.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2017 by Rocky Bernstein
+#  Copyright (c) 2016-2018 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -14,7 +14,8 @@ import sys
 
 from uncompyle6 import PYTHON3, IS_PYPY
 from uncompyle6.scanners.tok import Token
-from xdis.bytecode import op_size
+import xdis
+from xdis.bytecode import op_size, extended_arg_val
 from xdis.magics import py_str2float, canonic_python_version
 from xdis.util import code2num
 
@@ -103,14 +104,13 @@ class Scanner(object):
             target += pos + 3
         return target
 
-    # FIXME: the below can be removed after xdis version 3.6.1 has been released
-    def extended_arg_val(self, val):
-        return val << self.opc.EXTENDED_ARG_SHIFT
-
     def get_argument(self, pos):
         arg = self.code[pos+1] + self.code[pos+2] * 256
         return arg
 
+    def next_offset(self, op, offset):
+        return xdis.next_offset(op, self.opc, offset)
+
     def print_bytecode(self):
         for i in self.op_range(0, len(self.code)):
             op = self.code[i]
@@ -188,7 +188,7 @@ class Scanner(object):
 
             if op == self.opc.EXTENDED_ARG:
                 arg = code2num(code, offset+1) | extended_arg
-                extended_arg = self.extended_arg_val(arg)
+                extended_arg = extended_arg_val(self.opc, arg)
                 continue
 
             if op in instr:
@@ -240,7 +240,7 @@ class Scanner(object):
 
             if op == self.opc.EXTENDED_ARG:
                 arg = code2num(code, offset+1) | extended_arg
-                extended_arg = self.extended_arg_val(arg)
+                extended_arg = extended_arg_val(self.opc, arg)
                 continue
 
             if op in instr:

uncompyle6/semantics/fragments.py

@@ -56,6 +56,7 @@ from xdis.code import iscode
 from uncompyle6.semantics import pysource
 from uncompyle6 import parser
 from uncompyle6.scanner import Token, Code, get_scanner
+import uncompyle6.parser as python_parser
 from uncompyle6.semantics.check_ast import checker
 
 from uncompyle6.show import (
@@ -70,7 +71,7 @@ from uncompyle6.semantics.pysource import (
 
 from uncompyle6.semantics.consts import (
     INDENT_PER_LEVEL, NONE, PRECEDENCE,
-    TABLE_DIRECT, escape, minint, MAP
+    TABLE_DIRECT, escape, MAP, PASS
     )
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -986,17 +987,28 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.name = old_name
         self.return_none = rn
 
-    def build_ast(self, tokens, customize, is_lambda=False, noneInNames=False):
+    def build_ast(self, tokens, customize, is_lambda=False,
-        # assert type(tokens) == ListType
+                  noneInNames=False, isTopLevel=False):
+
+        # FIXME: DRY with pysource.py
+
         # assert isinstance(tokens[0], Token)
 
         if is_lambda:
+            for t in tokens:
+                if t.kind == 'RETURN_END_IF':
+                    t.kind = 'RETURN_END_IF_LAMBDA'
+                elif t.kind == 'RETURN_VALUE':
+                    t.kind = 'RETURN_VALUE_LAMBDA'
             tokens.append(Token('LAMBDA_MARKER'))
             try:
-                ast = parser.parse(self.p, tokens, customize)
+                # FIXME: have p.insts update in a better way
-            except parser.ParserError(e):
+                # modularity is broken here
-                raise ParserError(e, tokens)
+                p_insts = self.p.insts
-            except AssertionError(e):
+                self.p.insts = self.scanner.insts
+                ast = python_parser.parse(self.p, tokens, customize)
+                self.p.insts = p_insts
+            except (parser.ParserError(e), AssertionError(e)):
                 raise ParserError(e, tokens)
             maybe_show_ast(self.showast, ast)
             return ast
@@ -1011,19 +1023,30 @@ class FragmentsWalker(pysource.SourceWalker, object):
         #
         # NOTE: this differs from behavior in pysource.py
 
-        if len(tokens) >= 2 and not noneInNames:
+        if self.hide_internal:
-            if tokens[-1].kind == 'RETURN_VALUE':
+            if len(tokens) >= 2 and not noneInNames:
-                if tokens[-2].kind != 'LOAD_CONST':
+                if tokens[-1].kind in ('RETURN_VALUE', 'RETURN_VALUE_LAMBDA'):
-                    tokens.append(Token('RETURN_LAST'))
+                    # Python 3.4's classes can add a "return None" which is
-        if len(tokens) == 0:
+                    # invalid syntax.
-            return
+                    if tokens[-2].kind == 'LOAD_CONST':
+                        if isTopLevel or tokens[-2].pattr is None:
+                            del tokens[-2:]
+                        else:
+                            tokens.append(Token('RETURN_LAST'))
+                    else:
+                        tokens.append(Token('RETURN_LAST'))
+            if len(tokens) == 0:
+                return PASS
 
         # Build AST from disassembly.
         try:
+            # FIXME: have p.insts update in a better way
+            # modularity is broken here
+            p_insts = self.p.insts
+            self.p.insts = self.scanner.insts
             ast = parser.parse(self.p, tokens, customize)
-        except parser.ParserError(e):
+            self.p.insts = p_insts
-            raise ParserError(e, tokens)
+        except (parser.ParserError(e), AssertionError(e)):
-        except AssertionError(e):
             raise ParserError(e, tokens)
 
         maybe_show_ast(self.showast, ast)
@@ -1303,6 +1326,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         sep = INDENT_PER_LEVEL[:-1]
         start = len(self.f.getvalue())
         self.write('{')
+        self.set_pos_info(node[0], start, start+1)
 
         if self.version > 3.0:
             if node[0].kind.startswith('kvlist'):
@@ -1371,9 +1395,6 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 sep = line_seperator
         self.write('}')
         finish = len(self.f.getvalue())
-        for n in node:
-            n.parent = node
-            self.set_pos_info(n, start, finish)
         self.set_pos_info(node, start, finish)
         self.indent_less(INDENT_PER_LEVEL)
         self.prec = p
@@ -1626,7 +1647,8 @@ class FragmentsWalker(pysource.SourceWalker, object):
     pass
 
 def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
-                 showgrammar=False, is_pypy=False, walker=FragmentsWalker):
+                 showgrammar=False, code_objects={}, compile_mode='exec',
+                 is_pypy=False, walker=FragmentsWalker):
     """
     Convert the code object co into a python source fragment.
 
@@ -1654,7 +1676,8 @@ def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
     # store final output stream for case of error
     scanner = get_scanner(version, is_pypy=is_pypy)
 
-    tokens, customize = scanner.ingest(co)
+    tokens, customize = scanner.ingest(co, code_objects=code_objects,
+                                       show_asm=showasm)
 
     tokens, customize = scanner.ingest(co)
     maybe_show_asm(showasm, tokens)
@@ -1667,7 +1690,8 @@ def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
     #  Build AST from disassembly.
     # deparsed = pysource.FragmentsWalker(out, scanner, showast=showast)
     deparsed = walker(version, scanner, showast=showast,
-                      debug_parser=debug_parser)
+                      debug_parser=debug_parser, compile_mode=compile_mode,
+                      is_pypy=is_pypy)
 
     deparsed.ast = deparsed.build_ast(tokens, customize)
 
@@ -1771,7 +1795,7 @@ if __name__ == '__main__':
         return
 
     def deparse_test_around(offset, name, co, is_pypy=IS_PYPY):
-        sys_version = sys.version_info.major + (sys.version_info.minor / 10.0)
+        sys_version = sys.version_info[0] + (sys.version_info[1] / 10.0)
         walk = deparse_code_around_offset(name, offset, sys_version, co, showasm=False, showast=False,
                             showgrammar=False, is_pypy=IS_PYPY)
         print("deparsed source")
@@ -1800,6 +1824,8 @@ if __name__ == '__main__':
         return
 
     def get_code_for_fn(fn):
+        if hasattr(fn, 'func_code'):
+            return fn.func_code
         return fn.__code__
 
     def test():
@@ -1822,5 +1848,5 @@ if __name__ == '__main__':
     # deparse_test(get_code_for_fn(FragmentsWalker.fixup_offsets))
     # deparse_test(get_code_for_fn(FragmentsWalker.n_list))
     print('=' * 30)
-    deparse_test_around(408, 'n_list', get_code_for_fn(FragmentsWalker.n_build_list))
+    deparse_test_around(408, 'n_list', get_code_for_fn(FragmentsWalker.n_list))
     # deparse_test(inspect.currentframe().f_code)

uncompyle6/semantics/pysource.py

@@ -2540,6 +2540,8 @@ class SourceWalker(GenericASTTraversal, object):
     def build_ast(self, tokens, customize, is_lambda=False,
                   noneInNames=False, isTopLevel=False):
 
+        # FIXME: DRY with fragments.py
+
         # assert isinstance(tokens[0], Token)
 
         if is_lambda: