pydisassemble disassemble without grammar mangling

Some other small cleanups as well

uncompyle6/bin/pydisassemble.py

@@ -6,6 +6,10 @@
 from __future__ import print_function
 import sys, os, getopt
 
+from uncompyle6 import check_python_version
+from uncompyle6.disas import disassemble_files
+from uncompyle6.version import VERSION
+
 program, ext = os.path.splitext(os.path.basename(__file__))
 
 __doc__ = """
@@ -32,10 +36,6 @@ def main():
     Usage_short = \
     "%s [--help] [--verify] [--showasm] [--showast] [-o <path>] FILE|DIR..." % program
 
-    from uncompyle6 import check_python_version
-    from uncompyle6.disas import disassemble_files
-    from uncompyle6.version import VERSION
-
     check_python_version(program)
 
     outfile = '-'
@@ -85,7 +85,7 @@ def main():
     elif outfile and len(files) > 1:
         out_base = outfile; outfile = None
 
-    disassemble_files(src_base, out_base, files, outfile)
+    disassemble_files(src_base, out_base, files, outfile, True)
     return
 
 if __name__ == '__main__':

uncompyle6/disas.py

@@ -25,7 +25,7 @@ from uncompyle6.code import iscode
 from uncompyle6.load import check_object_path, load_module
 from uncompyle6.scanner import get_scanner
 
-def disco(version, co, out=None):
+def disco(version, co, out=None, native=False):
     """
     diassembles and deparses a given code block 'co'
     """
@@ -40,14 +40,17 @@ def disco(version, co, out=None):
               file=real_out)
 
     scanner = get_scanner(version)
-    tokens, customize = scanner.disassemble(co)
+    if native and hasattr(scanner, 'disassemble_native'):
+        tokens, customize = scanner.disassemble_native(co, True)
+    else:
+        tokens, customize = scanner.disassemble(co)
 
     for t in tokens:
         print(t, file=real_out)
     print(file=out)
 
 
-def disassemble_file(filename, outstream=None):
+def disassemble_file(filename, outstream=None, native=False):
     """
     disassemble Python byte-code file (.pyc)
 
@@ -58,12 +61,13 @@ def disassemble_file(filename, outstream=None):
     version, timestamp, magic_int, co = load_module(filename)
     if type(co) == list:
         for con in co:
-            disco(version, con, outstream)
+            disco(version, con, outstream, native)
     else:
-        disco(version, co, outstream)
+        disco(version, co, outstream, native)
     co = None
 
-def disassemble_files(in_base, out_base, files, outfile=None):
+def disassemble_files(in_base, out_base, files, outfile=None,
+                      native=False):
     """
     in_base	base directory for input files
     out_base	base directory for output files (ignored when
@@ -110,7 +114,7 @@ def disassemble_files(in_base, out_base, files, outfile=None):
 
         # try to decomyple the input file
         try:
-            disassemble_file(infile, outstream)
+            disassemble_file(infile, outstream, native)
         except KeyboardInterrupt:
             if outfile:
                 outstream.close()
@@ -146,7 +150,7 @@ def _test():
             sys.exit(2)
     else:
         fn = sys.argv[1]
-    disassemble_file(fn)
+    disassemble_file(fn, native=True)
 
 if __name__ == "__main__":
     _test()

uncompyle6/parser.py

@@ -513,7 +513,8 @@ def python_parser(version, co, out=sys.stdout, showasm=False,
             print(t)
 
     # For heavy grammar debugging
-    parser_debug = {'rules': True, 'transition': True, 'reduce' : True}
+    parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
+                    'showstack': 'full'}
     p = get_python_parser(version, parser_debug)
     return parse(p, tokens, customize)
 

uncompyle6/parsers/parse3.py

@@ -582,6 +582,7 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, opname, token.attr, customize)
         return
 
+
 class Python32Parser(Python3Parser):
     def p_32(self, args):
         """
@@ -596,15 +597,6 @@ class Python34Parser(Python3Parser):
         _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
         """
 
-class Python3ParserSingle(Python3Parser, PythonParserSingle):
-    pass
-
-class Python32ParserSingle(Python32Parser, PythonParserSingle):
-    pass
-
-class Python34ParserSingle(Python34Parser, PythonParserSingle):
-    pass
-
 class Python35onParser(Python3Parser):
     def p_35on(self, args):
         """
@@ -618,5 +610,17 @@ class Python35onParser(Python3Parser):
                 WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
         """
 
+class Python3ParserSingle(Python3Parser, PythonParserSingle):
+    pass
+
+
+class Python32ParserSingle(Python32Parser, PythonParserSingle):
+    pass
+
+
+class Python34ParserSingle(Python34Parser, PythonParserSingle):
+    pass
+
+
 class Python35onParserSingle(Python35onParser, PythonParserSingle):
     pass

uncompyle6/scanners/scanner3.py

@@ -40,10 +40,13 @@ import uncompyle6.scanner as scan
 
 class Scanner3(scan.Scanner):
 
+    ## FIXME opnames should be passed in here
     def __init__(self, version):
         self.version = version
         scan.Scanner.__init__(self, version)
 
+
+    ## FIXME opnames should be moved to init
     def disassemble3(self, co, opnames, classname=None, code_objects={}):
 
         # import dis; dis.disassemble(co) # DEBUG
@@ -51,24 +54,12 @@ class Scanner3(scan.Scanner):
         # Container for tokens
         tokens = []
 
-        customize = {}
         self.code = array('B', co.co_code)
         self.build_lines_data(co)
         self.build_prev_op()
 
         bytecode = dis3.Bytecode(co, opnames)
 
-        # self.lines contains (block,addrLastInstr)
-        if classname:
-            classname = '_' + classname.lstrip('_') + '__'
-
-            def unmangle(name):
-                if name.startswith(classname) and name[-2:] != '__':
-                    return name[len(classname) - 2:]
-                return name
-        else:
-            pass
-
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT' for those
         # assertions
@@ -131,7 +122,7 @@ class Scanner3(scan.Scanner):
                     opname = 'MAKE_FUNCTION_N%d' % name_pair_args
                     pass
                 if annotate_args > 0:
-                    opname = '%s_A_%d' % [op_name, annotate_args]
+                    opname = '%s_A_%d' % [opname, annotate_args]
                     pass
                 opname = '%s_%d' % (opname, pos_args)
                 pattr = ("%d positional, %d keyword pair, %d annotated" %
@@ -150,16 +141,14 @@ class Scanner3(scan.Scanner):
                             'RAISE_VARARGS'
                             ):
                 pos_args = inst.argval
-                if inst.opname != 'BUILD_SLICE':
-                    customize[opname] = pos_args
-                    pass
                 opname = '%s_%d' % (opname, pos_args)
             elif opname == 'JUMP_ABSOLUTE':
                 pattr = inst.argval
                 target = self.get_target(inst.offset)
                 if target < inst.offset:
+                    next_opname = opnames[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
-                        self.code[inst.offset+3] not in (END_FINALLY, POP_BLOCK)
+                        next_opname not in ('END_FINALLY', 'POP_BLOCK')
                         and inst.offset not in self.not_continue):
                         opname = 'CONTINUE'
                     else:
@@ -180,6 +169,32 @@ class Scanner3(scan.Scanner):
             pass
         return tokens, {}
 
+    def disassemble3_native(self, co, opnames, classname=None, code_objects={}):
+        """
+        Like disassemble3 but doesn't try to adjust any opcodes.
+        """
+        # Container for tokens
+        tokens = []
+
+        self.code = array('B', co.co_code)
+
+        bytecode = dis3.Bytecode(co, opnames)
+
+        for inst in bytecode:
+            pattr =  inst.argrepr
+            opname = inst.opname
+            tokens.append(
+                Token(
+                    type_ = opname,
+                    attr = inst.argval,
+                    pattr = pattr,
+                    offset = inst.offset,
+                    linestart = inst.starts_line,
+                    )
+                )
+            pass
+        return tokens, {}
+
     def disassemble_generic(self, co, classname=None, code_objects={}):
         """
         Convert code object <co> into a sequence of tokens.

uncompyle6/scanners/scanner34.py

@@ -8,16 +8,19 @@ scanner routine for Python 3.
 
 from __future__ import print_function
 
-import uncompyle6.scanners.scanner3 as scan3
+from uncompyle6.scanners.scanner3 import Scanner3
 from uncompyle6.opcodes.opcode_34 import opname as opnames
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from uncompyle6.opcodes.opcode_34 import JUMP_OPs
 
-class Scanner34(scan3.Scanner3):
+class Scanner34(Scanner3):
     def disassemble(self, co, classname=None, code_objects={}):
         return self.disassemble3(co, opnames, classname, code_objects)
 
+    def disassemble_native(self, co, classname=None, code_objects={}):
+        return self.disassemble3_native(co, opnames, classname, code_objects)
+
 if __name__ == "__main__":
     import inspect
     co = inspect.currentframe().f_code

uncompyle6/scanners/scanner35.py

@@ -8,16 +8,19 @@ scanner routine for Python 3.
 
 from __future__ import print_function
 
-import uncompyle6.scanners.scanner3 as scan3
+from uncompyle6.scanners.scanner3 import Scanner3
 from uncompyle6.opcodes.opcode_35 import opname as opnames
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from uncompyle6.opcodes.opcode_35 import JUMP_OPs
 
-class Scanner35(scan3.Scanner3):
+class Scanner35(Scanner3):
     def disassemble(self, co, classname=None, code_objects={}):
         return self.disassemble3(co, opnames, classname, code_objects)
 
+    def disassemble_native(self, co, classname=None, code_objects={}):
+        return self.disassemble3_native(co, opnames, classname, code_objects)
+
 if __name__ == "__main__":
     import inspect
     co = inspect.currentframe().f_code

uncompyle6/scanners/tok.py

@@ -37,10 +37,9 @@ class Token:
 
     def __str__(self):
         pattr = self.pattr if self.pattr is not None else ''
-        if self.linestart:
+        prefix = '\n%4d  ' % self.linestart if self.linestart else (' ' * 6)
-            return '\n%4d  %6s\t%-17s %r' % (self.linestart, self.offset, self.type, pattr)
+        return (prefix +
-        else:
+                ('%6s\t%-17s %r' % (self.offset, self.type, pattr)))
-            return '      %6s\t%-17s %r' % (self.offset, self.type, pattr)
 
     def __hash__(self):
         return hash(self.type)