Start 3.4 more stringent disassembly testing. Disassembly format has

changed slightly. misc small bugs.

.gitignore

@@ -1,8 +1,10 @@
-*.pyc
 *_dis
 *~
+/.cache
 /.eggs
 /.python-version
+/__pkginfo__.pyc
 /dist
 /uncompyle6.egg-info
+__pycache__
 build

pytest/.gitignore

@@ -0,0 +1 @@
+/__pycache__

pytest/testdata/test1.right

@@ -18,7 +18,7 @@
   19      24	LOAD_CONST        'Topic :: Software Development :: Debuggers'
 
   20      27	LOAD_CONST        'Topic :: Software Development :: Libraries :: Python Modules'
-          30	BUILD_LIST_6      None
+          30	BUILD_LIST_6      ''
           33	STORE_NAME        'classifiers'
 
   24      36	LOAD_CONST        'Rocky Bernstein'
@@ -27,7 +27,7 @@
   25      42	LOAD_CONST        'rb@dustyfeet.com'
           45	STORE_NAME        'author_email'
 
-  26      48	LOAD_CONST        None
+  26      48	LOAD_CONST        ''
           51	STORE_NAME        'ftp_url'
 
   28      54	LOAD_CONST        'python-debugger@googlegroups.com'
@@ -38,10 +38,10 @@
 
   30      66	LOAD_CONST        'uncompyle6'
           69	LOAD_CONST        'uncompyle6.opcodes'
-          72	BUILD_LIST_2      None
+          72	BUILD_LIST_2      ''
           75	STORE_NAME        'packages'
 
-  31      78	LOAD_CONST        None
+  31      78	LOAD_CONST        ''
           81	STORE_NAME        'py_modules'
 
   32      84	LOAD_CONST        'Python byte-code disassembler and source-code converter'
@@ -49,19 +49,19 @@
 
   33      90	LOAD_CONST        'bin/uncompyle6'
           93	LOAD_CONST        'bin/pydisassemble'
-          96	BUILD_LIST_2      None
+          96	BUILD_LIST_2      ''
           99	STORE_NAME        'scripts'
 
   35     102	LOAD_CONST        -1
-         105	LOAD_CONST        None
+         105	LOAD_CONST        ''
          108	IMPORT_NAME       'os.path'
          111	STORE_NAME        'os'
 
   38     114	LOAD_CONST        '<code_object get_srcdir>'
-         117	MAKE_FUNCTION_0   None
+         117	MAKE_FUNCTION_0   ''
          120	STORE_NAME        'get_srcdir'
 
-  43     123	BUILD_MAP         None
+  43     123	BUILD_MAP         ''
          126	STORE_NAME        'ns'
 
   44     129	LOAD_CONST        '2.0'
@@ -74,15 +74,15 @@
          144	STORE_NAME        'zip_safe'
 
   51     147	LOAD_CONST        '<code_object read>'
-         150	MAKE_FUNCTION_0   None
+         150	MAKE_FUNCTION_0   ''
          153	STORE_NAME        'read'
 
   54     156	LOAD_NAME         'read'
          159	LOAD_CONST        'README.rst'
-         162	CALL_FUNCTION_1   None
+         162	CALL_FUNCTION_1   ''
          165	LOAD_CONST        '\n'
-         168	BINARY_ADD        None
+         168	BINARY_ADD        ''
          169	STORE_NAME        'long_description'
-         172	LOAD_CONST        None
+         172	LOAD_CONST        ''
-         175	RETURN_VALUE      None
+         175	RETURN_VALUE      ''
 

pytest/testdata/test_import_25.right

@@ -5,25 +5,25 @@
            3	STORE_NAME        '__doc__'
 
   11       6	LOAD_CONST        -1
-           9	LOAD_CONST        None
+           9	LOAD_CONST        ''
           12	IMPORT_NAME       'sys'
           15	STORE_NAME        'sys'
 
   12      18	LOAD_CONST        -1
-          21	LOAD_CONST        None
+          21	LOAD_CONST        ''
           24	IMPORT_NAME       'os'
           27	STORE_NAME        'os'
           30	LOAD_CONST        -1
-          33	LOAD_CONST        None
+          33	LOAD_CONST        ''
           36	IMPORT_NAME_CONT  'sys'
           39	STORE_NAME        'sys'
           42	LOAD_CONST        -1
-          45	LOAD_CONST        None
+          45	LOAD_CONST        ''
           48	IMPORT_NAME_CONT  'BaseHTTPServer'
           51	STORE_NAME        'BaseHTTPServer'
 
   14      54	LOAD_CONST        -1
-          57	LOAD_CONST        None
+          57	LOAD_CONST        ''
           60	IMPORT_NAME       'test.test_MimeWriter'
           63	STORE_NAME        'test'
 
@@ -32,7 +32,7 @@
           72	IMPORT_NAME       'rfc822'
           75	IMPORT_FROM       'Message'
           78	STORE_NAME        'Message'
-          81	POP_TOP           None
+          81	POP_TOP           ''
 
   17      82	LOAD_CONST        -1
           85	LOAD_CONST        ('Message', 'decode', 'choose_boundary')
@@ -43,33 +43,33 @@
          100	STORE_NAME        'decode'
          103	IMPORT_FROM       'choose_boundary'
          106	STORE_NAME        'choose_boundary'
-         109	POP_TOP           None
+         109	POP_TOP           ''
 
   18     110	LOAD_CONST        -1
          113	LOAD_CONST        ('*',)
          116	IMPORT_NAME       'os'
-         119	IMPORT_STAR       None
+         119	IMPORT_STAR       ''
 
   20     120	SETUP_LOOP        '162'
          123	LOAD_NAME         'globals'
-         126	CALL_FUNCTION_0   None
+         126	CALL_FUNCTION_0   ''
          129	LOAD_ATTR         'items'
-         132	CALL_FUNCTION_0   None
+         132	CALL_FUNCTION_0   ''
-         135	GET_ITER          None
+         135	GET_ITER          ''
          136	FOR_ITER          '161'
-         139	UNPACK_SEQUENCE_2 None
+         139	UNPACK_SEQUENCE_2 ''
          142	STORE_NAME        'k'
          145	STORE_NAME        'v'
 
   21     148	LOAD_NAME         'k'
-         151	UNARY_CONVERT     None
+         151	UNARY_CONVERT     ''
-         152	PRINT_ITEM        None
+         152	PRINT_ITEM        ''
          153	LOAD_NAME         'v'
-         156	PRINT_ITEM_CONT   None
+         156	PRINT_ITEM_CONT   ''
-         157	PRINT_NEWLINE_CONT None
+         157	PRINT_NEWLINE_CONT ''
          158	JUMP_BACK         '136'
-         161	POP_BLOCK         None
+         161	POP_BLOCK         ''
        162_0	COME_FROM         '120'
-         162	LOAD_CONST        None
+         162	LOAD_CONST        ''
-         165	RETURN_VALUE      None
+         165	RETURN_VALUE      ''
 

test/compile_tests

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.3
+#!/usr/bin/env python
 """
 compile_tests -- compile test patterns for the decompyle test suite
 
@@ -9,6 +9,8 @@ See http://www.crazy-compilers.com/decompyle/ for
 for further information
 """
 
+from __future__ import print_function
+
 import py_compile, os, sys, getopt
 
 work_dir = os.path.dirname(sys.argv[0])
@@ -26,8 +28,8 @@ for opt, val in opts:
 if args:
     raise 'This tool does not want any arguments'
 
-print "Using files in dir %s" % src_dir
+print("Using files in dir %s" % src_dir)
-print "Compiling into dir %s" % work_dir
+print( "Compiling into dir %s" % work_dir)
 
 tests = {}
 
@@ -51,30 +53,44 @@ tests['2.2'] = ['divide_future', 'divide_no_future', 'iterators',
 tests['2.3'] = tests['2.2']
 tests['2.5'] = tests['2.3']
 tests['2.6'] = tests['2.5']
-tests['2.7'] = ['mine'] + tests['2.6']
+# tests['2.7'] = ['mine'] + tests['2.6']
+tests['2.7'] = [
+    'source_3.4/call_arguments/keyword',
+    'source_3.4/call_arguments/positional'
+    ]
+
+tests['3.4'] = [
+    # 'source_3.4/branching/ifelse',
+    # 'source_3.4/branching/if'
+    'source_3.4/call_arguments/keyword',
+    'source_3.4/call_arguments/positional'
+    ]
+
 total_tests = len(tests['2.7'])
 #tests['2.2'].sort(); print tests['2.2']
 
 extension = '.py' + (__debug__ and 'c' or 'o')
 
-def compile(file, target_dir):
+def compile(filename, target_dir):
-    sfile = os.path.join(src_dir,    'test_%s.py' % file)
+    sfile = os.path.join(src_dir,    '%s.py' % filename)
-    cfile = os.path.join(target_dir, 'test_%s%s' % (file,  extension) )
+    cfile = os.path.join(target_dir, '%s%s'
+                         % (os.path.basename(filename),  extension) )
     py_compile.compile(sfile, cfile=cfile)
 
 def compile_for_version(version):
     target_dir = os.path.join(work_dir, 'bytecode_' + version)
     if not os.path.exists(target_dir):
         os.mkdir(target_dir)
-    for file in tests[version]:
+    for filename in tests[version]:
-        compile(file, target_dir)
+        compile(filename, target_dir)
 
 try:
     version = '%i.%i' % sys.version_info[:2]
 except AttributeError:
     version = sys.version[:3]
 
-print 'Compiling test files for Python', version,
+print('Compiling test files for Python', version)
-print '(%i/%i files)' % (len(tests[version]), total_tests)
+# print('(%i/%i files)' % (len(tests[version]), total_tests))
+print('%i files' % len(tests[version]))
 compile_for_version(version)
-print 'Done.'
+print('Done.')

test/dis-compare.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# Mode: -*- python -*-
+#
+# Copyright (c) 2015 by Rocky Bernstein <rb@dustyfeet.com>
+#
+from __future__ import print_function
+
+
+import dis, os.path, sys
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+program =  os.path.basename(__file__)
+
+__doc__ = """
+Usage: {0} [OPTIONS]... FILE
+
+""".format(program)
+
+usage_short = "Usage: {0} [OPTIONS]... FILE".format(program)
+
+import uncompyle6
+from uncompyle6.disas import disco
+
+version = sys.version[:3]
+
+def inst_fmt(inst):
+    if inst.starts_line:
+        return '\n%4d  %6s\t%-17s %r' % (inst.starts_line, inst.offset, inst.opname,
+                                         inst.argrepr)
+    else:
+        return '      %6s\t%-17s %r' % (inst.offset, inst.opname, inst.argrepr)
+
+    print
+    return
+
+def compare_ok(version, co):
+    out  = StringIO()
+    if version == 2.7:
+        dis.disco(co)
+        return
+
+    bytecode = dis.Bytecode(co)
+    disco(version, co, out)
+    got_lines = out.getvalue().split("\n")[2:]
+    i = 0
+    good_lines = "\n".join([inst_fmt(inst) for inst in bytecode]).split("\n")
+    for good_line in good_lines:
+        if '\tCOME_FROM         ' in got_lines[i]:
+            i += 1
+        if '\tJUMP_FORWARD      ' in good_line:
+            good_line = good_line = good_line[:32] + good_line[35:]
+
+        if got_lines[i] != good_line:
+            print('line %d %s' % (i+1, ('=' * 30)))
+            print(good_line)
+            print("vs %s" % ('-' * 10))
+            print(got_lines[i])
+            return False
+        i += 1
+    return True
+
+if version != '2.7' and version != '3.4':
+    print('Error:  {0} requires Python 2.7 or 3.4.'.format(program),
+          file=sys.stderr)
+    sys.exit(-1)
+
+# if len(sys.argv) != 2:
+#     print(usage_short, file=sys.stderr)
+#     sys.exit(1)
+
+# filename = sys.arv[1]
+def get_srcdir():
+    filename = os.path.normcase(os.path.dirname(__file__))
+    return os.path.realpath(filename)
+
+src_dir = get_srcdir()
+os.chdir(src_dir)
+
+files=[
+    'if',
+    'ifelse',
+    # 'keyword',
+    ]
+
+for base in files:
+    filename = "bytecode_%s/%s.pyc" % (version, base)
+    version, co = uncompyle6.load_module(filename)
+    ok = True
+    if type(co) == list:
+        for con in co:
+            ok = compare_ok(version, con)
+            if not ok: break
+    else:
+        ok = compare_ok(version, co)
+    if ok:
+        print("Disassembly of %s checks out!" % filename)
+    else:
+        print("Disassembly of %s mismatches." % filename)
+        break

test/simple-uncompyle-code-test.py

@@ -2,10 +2,8 @@
 
 from __future__ import print_function
 
-import uncompyle6
+from uncompyle6 import uncompyle
-from uncompyle6 import uncompyle, walker, verify, magics
+import sys, inspect
-from uncompyle6.spark import GenericASTTraversal, GenericASTTraversalPruningException
-import sys, inspect, types
 
 if (sys.version_info > (3, 0)):
     from io import StringIO

test/source_3.4/branching/if.py

@@ -0,0 +1,2 @@
+if a:
+    b = c

test/source_3.4/branching/ifelse.py

@@ -0,0 +1,4 @@
+if a:
+    b = c
+else:
+    d = e

uncompyle6/.gitignore

@@ -0,0 +1 @@
+*.pyc

uncompyle6/scanner.py

@@ -1,23 +1,30 @@
-from __future__ import print_function
-
 #  Copyright (c) 1999 John Aycock
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #
 #  See main module for license.
 #
+"""
+scanner/disassembler module. From here we call various verison-specific
+scanners, e.g. for Python 2.7 or 3.4.
+
+This overlaps Python's dis module, but it can be run from Python 2 or
+Python 3 and other versions of Python. Also, we save token information
+for later use in deparsing.
+"""
+
+from __future__ import print_function
+
 
 __all__ = ['Token', 'Scanner', 'Code']
 
-import sys, types
+import sys
-from collections import namedtuple
-from array import array
-from operator import itemgetter
 
 if (sys.version_info > (3, 0)):
     intern = sys.intern
     L65536 = 65536
-    import uncompyle6
+    def cmp(a, b):
+        return (a > b) - (a < b)
 else:
     L65536 = long(65536)
 
@@ -49,7 +56,7 @@ class Token:
         return str(self.type)
 
     def __str__(self):
-        pattr = self.pattr
+        pattr = self.pattr if self.pattr is not None else ''
         if self.linestart:
             return '\n%4d  %6s\t%-17s %r' % (self.linestart, self.offset, self.type, pattr)
         else:

uncompyle6/scanners/scanner25.py

@@ -1,16 +1,16 @@
+
+#  Copyright (c) 1999 John Aycock
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2015 by Rocky Bernstein
+#
+#  See main module for license.
 """
-  Python 2.5 bytecode scanner/deparser
+Python 2.5 bytecode scanner/deparser
-
-  Copyright (c) 1999 John Aycock
-  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
-  Copyright (c) 2015 by Rocky Bernstein
-
-  See main module for license.
 
 This overlaps Python's 2.5's dis module, but it can be run from
-Python3 and other versions of Python. Also, we save token information
+Python 3 and other versions of Python. Also, we save token
-for later use in deparsing.
+information for later use in deparsing.
 """
 
 import types

uncompyle6/scanners/scanner26.py

@@ -1,14 +1,14 @@
+#  Copyright (c) 1999 John Aycock
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2015 by Rocky Bernstein
+#
+#  See main module for license.
+
 """
-  Python 2.6 bytecode scanner
+Python 2.6 bytecode scanner
 
-  Copyright (c) 1999 John Aycock
+This overlaps Python's 2.6's dis module, but it can be run from Python 3 and
-  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
-  Copyright (c) 2015 by Rocky Bernstein
-
-  See main module for license.
-
-This overlaps Python's 2.6's dis module, but it can be run from Python3 and
 other versions of Python. Also, we save token information for later
 use in deparsing.
 """

uncompyle6/scanners/scanner27.py

@@ -1,18 +1,16 @@
+# Copyright (c) 1999 John Aycock
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2015 by Rocky Bernstein
 """
-  Python 2.5 bytecode scanner/deparser
+Python 2.7 bytecode scanner/deparser
-
-  Copyright (c) 1999 John Aycock
-  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
-  Copyright (c) 2015 by Rocky Bernstein
-
-  See main module for license.
 
 This overlaps Python's 2.7's dis module, but it can be run from
-Python3 and other versions of Python. Also, we save token information
+Python 3 and other versions of Python. Also, we save token information
 for later use in deparsing.
 """
 
+
 from __future__ import print_function
 
 import dis, types

uncompyle6/scanners/scanner32.py

@@ -1,15 +1,14 @@
+#  Copyright (c) 1999 John Aycock
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2015 by Rocky Bernstein
+#
+#  See main module for license.
 """
-  Python 3.2 bytecode scanner/deparser
+Python 3.2 bytecode scanner/deparser
-
-  Copyright (c) 1999 John Aycock
-  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
-  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
-  Copyright (c) 2015 by Rocky Bernstein
-
-  See main module for license.
 
 This overlaps Python's 3.2's dis module, but it can be run from
-Python3 and other versions of Python. Also, we save token information
+Python 2 and other versions of Python. Also, we save token information
 for later use in deparsing.
 """
 

uncompyle6/scanners/scanner34.py

@@ -1,24 +1,27 @@
+#  Copyright (c) 1999 John Aycock
+#  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+#  Copyright (c) 2015 by Rocky Bernstein
+#
+#  See main module for license.
 """
-  Python 2.5 bytecode scanner/deparser
+Python 3.4 bytecode scanner/deparser
 
-  Copyright (c) 1999 John Aycock
+This overlaps Python's 3.4's dis module, but it can be run from
-  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+Python 2 and other versions of Python. Also, we save token information
-  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
+for later use in deparsing.
-  Copyright (c) 2015 by Rocky Bernstein
-
-  See main module for license.
 """
 
 from __future__ import print_function
 
-import dis, marshal
+import dis
 from collections import namedtuple
 
 from uncompyle6.scanner import Token, L65536
 
 # Get all the opcodes into globals
 globals().update(dis.opmap)
-from uncompyle6.opcodes.opcode_27 import *
+from uncompyle6.opcodes.opcode_34 import *
 import uncompyle6.scanner as scan
 
 
@@ -26,11 +29,6 @@ class Scanner34(scan.Scanner):
     def __init__(self):
         self.Token = scan.Scanner.__init__(self, 3.4) # check
 
-    def run(self, bytecode):
-        code_object = marshal.loads(bytecode)
-        tokens = self.tokenize(code_object)
-        return tokens
-
     def disassemble(self, co):
         """
         Convert code object <co> into a sequence of tokens.