From 217b1b6f5494f57d2f7d6de476e6a1e772612aa1 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 22 Dec 2015 05:15:00 -0500 Subject: [PATCH] main.py, pysource.py DRY deparse_code from main. Is better on showing exception errrors such as when a pyc file is not found uncompyle6: Hook in --grammar option to showing grammar. rules. Default now does not show timestamp. --- bin/uncompyle6 | 7 ++-- uncompyle6/main.py | 56 ++++++++++---------------------- uncompyle6/parser.py | 13 ++++---- uncompyle6/parsers/parse2.py | 6 ++-- uncompyle6/parsers/spark.py | 2 +- uncompyle6/semantics/pysource.py | 39 +++++++++++----------- 6 files changed, 53 insertions(+), 70 deletions(-) diff --git a/bin/uncompyle6 b/bin/uncompyle6 index b1b71cc7..f73a639b 100755 --- a/bin/uncompyle6 +++ b/bin/uncompyle6 @@ -24,7 +24,7 @@ Options: uncompyle6 -o /tmp /usr/lib/python1.5 -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis -c attempts a disassembly after compiling - -d do not print timestamps + -d print timestamps -p use number of processes -r recurse directories looking for .pyc and .pyo files --verify compare generated source with input byte-code @@ -64,7 +64,7 @@ numproc = 0 outfile = '-' out_base = None codes = [] -timestamp = True +timestamp = False timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" try: @@ -93,7 +93,7 @@ for opt, val in opts: elif opt == '-o': outfile = val elif opt == ('--timestamp', '-d'): - timestamp = False + timestamp = True elif opt == '-c': codes.append(val) elif opt == '-p': @@ -140,6 +140,7 @@ elif outfile and len(files) > 1: if timestamp: print(time.strftime(timestampfmt)) + if numproc <= 1: try: result = main(src_base, out_base, files, codes, outfile, diff --git a/uncompyle6/main.py b/uncompyle6/main.py index dbbcbc1b..45cae2b9 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -1,22 +1,18 @@ from __future__ import print_function -import os, sys, types +import os, sys from uncompyle6.disas import check_object_path from uncompyle6 import verify from uncompyle6.semantics import pysource -from uncompyle6.scanner import get_scanner from uncompyle6.load import load_module -# FIXME: remove duplicate code from deparse_code def uncompyle(version, co, out=None, showasm=False, showast=False, showgrammar=False): """ disassembles and deparses a given code block 'co' """ - assert isinstance(co, types.CodeType) - # store final output stream for case of error real_out = out or sys.stdout print('# Python %s' % version, file=real_out) @@ -24,43 +20,25 @@ def uncompyle(version, co, out=None, showasm=False, showast=False, print('# Embedded file name: %s' % co.co_filename, file=real_out) - scanner = get_scanner(version) - tokens, customize = scanner.disassemble(co) - - if showasm: - for t in tokens: - print(t, file=real_out) - print(file=out) - - # Build AST from disassembly. - walk = pysource.Walker(version, out, scanner, - showast=showast, showgrammar=showgrammar) try: - ast = walk.build_ast(tokens, customize) + deparsed = pysource.deparse_code(version, co, out, showasm, showast, showgrammar) except pysource.ParserError as e : # parser failed, dump disassembly print(e, file=real_out) raise - del tokens # save memory - # convert leading '__doc__ = "..." into doc string - assert ast == 'stmts' + # FIXME: remove duplicate code from deparse_code try: - if ast[0][0] == pysource.ASSIGN_DOC_STRING(co.co_consts[0]): - walk.print_docstring('', co.co_consts[0]) - del ast[0] - if ast[-1] == pysource.RETURN_NONE: - ast.pop() # remove last node + if deparsed.ast[0][0] == pysource.ASSIGN_DOC_STRING(co.co_consts[0]): + deparsed.print_docstring('', co.co_consts[0]) + del deparsed.ast[0] + if deparsed.ast[-1] == pysource.RETURN_NONE: + deparsed.ast.pop() # remove last node # todo: if empty, add 'pass' except: pass - walk.mod_globs = pysource.find_globals(ast, set()) - walk.gen_source(ast, customize) - for g in walk.mod_globs: - walk.write('global %s ## Warning: Unused global' % g) - if walk.ERROR: - raise walk.ERROR -def uncompyle_file(filename, outstream=None, showasm=False, showast=False): +def uncompyle_file(filename, outstream=None, showasm=False, showast=False, + showgrammar=False): """ decompile Python byte-code file (.pyc) """ @@ -68,14 +46,15 @@ def uncompyle_file(filename, outstream=None, showasm=False, showast=False): version, magic_int, co = load_module(filename) if type(co) == list: for con in co: - uncompyle(version, con, outstream, showasm, showast) + uncompyle(version, con, outstream, showasm, showast, showgrammar) else: - uncompyle(version, co, outstream, showasm, showast) + uncompyle(version, co, outstream, showasm, showast, showgrammar) co = None def main(in_base, out_base, files, codes, outfile=None, - showasm=False, showast=False, do_verify=False): - ''' + showasm=False, showast=False, do_verify=False, + showgrammar=False): + """ in_base base directory for input files out_base base directory for output files (ignored when files list of filenames to be uncompyled (relative to src_base) @@ -85,7 +64,7 @@ def main(in_base, out_base, files, codes, outfile=None, - outfile= (out_base is ignored) - files below out_base out_base=... - stdout out_base=None, outfile=None - ''' + """ def _get_outstream(outfile): dir = os.path.dirname(outfile) failed_file = outfile + '_failed' @@ -121,7 +100,7 @@ def main(in_base, out_base, files, codes, outfile=None, # Try to uncmpile the input file try: - uncompyle_file(infile, outstream, showasm, showast) + uncompyle_file(infile, outstream, showasm, showast, showgrammar) tot_files += 1 except ValueError as e: sys.stderr.write("\n# %s" % e) @@ -138,6 +117,7 @@ def main(in_base, out_base, files, codes, outfile=None, outstream.close() os.rename(outfile, outfile + '_failed') else: + sys.stderr.write("\n# %s" % sys.exc_info()[1]) sys.stderr.write("\n# Can't uncompile %s\n" % infile) else: # uncompile successfull if outfile: diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 36c4bbe0..12f30755 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -10,7 +10,7 @@ from __future__ import print_function import sys -from uncompyle6.parsers.spark import GenericASTBuilder +from uncompyle6.parsers.spark import GenericASTBuilder, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG class ParserError(Exception): def __init__(self, token, offset): @@ -75,19 +75,20 @@ def parse(p, tokens, customize): return ast -def get_python_parser(version): +def get_python_parser(version, debug_parser): """ Returns parser object for Python version 2 or 3 depending on the parameter passed. """ if version < 3.0: import uncompyle6.parsers.parse2 as parse2 - return parse2.Python2Parser() + return parse2.Python2Parser(debug_parser) else: import uncompyle6.parsers.parse3 as parse3 - return parse3.Python3Parser() + return parse3.Python3Parser(debug_parser) -def python_parser(version, co, out=sys.stdout, showasm=False): +def python_parser(version, co, out=sys.stdout, showasm=False, + parser_debug=PARSER_DEFAULT_DEBUG): import inspect assert inspect.iscode(co) from uncompyle6.scanner import get_scanner @@ -97,7 +98,7 @@ def python_parser(version, co, out=sys.stdout, showasm=False): # for t in tokens: # print(t) - p = get_python_parser(version) + p = get_python_parser(version, parser_debug) return parse(p, tokens, customize) if __name__ == '__main__': diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 1acb5fd2..a887b6e0 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -19,12 +19,12 @@ from __future__ import print_function from uncompyle6.parser import PythonParser, nop_func from uncompyle6.parsers.astnode import AST -from uncompyle6.parsers.spark import GenericASTBuilder +from uncompyle6.parsers.spark import GenericASTBuilder, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG class Python2Parser(PythonParser): - def __init__(self): - GenericASTBuilder.__init__(self, AST, 'stmts') + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + GenericASTBuilder.__init__(self, AST, 'stmts', debug=debug_parser) self.customized = {} def p_funcdef(self, args): diff --git a/uncompyle6/parsers/spark.py b/uncompyle6/parsers/spark.py index 743d20e1..029df1f5 100644 --- a/uncompyle6/parsers/spark.py +++ b/uncompyle6/parsers/spark.py @@ -643,7 +643,7 @@ class GenericParser: class GenericASTBuilder(GenericParser): def __init__(self, AST, start, debug=DEFAULT_DEBUG): - GenericParser.__init__(self, start) + GenericParser.__init__(self, start, debug=debug) self.AST = AST def preprocess(self, rule, func): diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 159a04de..cb27ac2c 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -470,7 +470,8 @@ def find_none(node): class Walker(GenericASTTraversal, object): stacked_params = ('f', 'indent', 'isLambda', '_globals') - def __init__(self, version, out, scanner, showast=False, showgrammar=False): + def __init__(self, version, out, scanner, showast=False, + debug_parser=PARSER_DEFAULT_DEBUG): GenericASTTraversal.__init__(self, ast=None) self.scanner = scanner params = { @@ -478,7 +479,8 @@ class Walker(GenericASTTraversal, object): 'indent': '', } self.version = version - self.p = get_python_parser(version) + self.p = get_python_parser(version, debug_parser=debug_parser) + self.debug_parser = dict(debug_parser) self.showast = showast self.params = params self.param_stack = [] @@ -1540,7 +1542,6 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, assert inspect.iscode(co) # store final output stream for case of error - __real_out = out or sys.stdout scanner = get_scanner(version) tokens, customize = scanner.disassemble(co) @@ -1548,37 +1549,37 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False, for t in tokens: print(t) - parser_debug = dict(PARSER_DEFAULT_DEBUG) - parser_debug['reduce'] = showgrammar + debug_parser = dict(PARSER_DEFAULT_DEBUG) + debug_parser['reduce'] = showgrammar # Build AST from disassembly. - walk = Walker(version, out, scanner, showast=showast) + deparsed = Walker(version, out, scanner, showast=showast, debug_parser=debug_parser) - try: - walk.ast = walk.build_ast(tokens, customize) - except ParserError as e : # parser failed, dump disassembly - print(e, file=__real_out) - raise + deparsed.ast = deparsed.build_ast(tokens, customize) + + assert deparsed.ast == 'stmts', 'Should have parsed grammar start' del tokens # save memory # convert leading '__doc__ = "..." into doc string - assert walk.ast == 'stmts' - walk.mod_globs = find_globals(walk.ast, set()) - walk.gen_source(walk.ast, customize) + deparsed.mod_globs = find_globals(deparsed.ast, set()) - for g in walk.mod_globs: - walk.write('global %s ## Warning: Unused global' % g) - if walk.ERROR: - raise walk.ERROR + # What we've been waiting for: Generate source from AST! + deparsed.gen_source(deparsed.ast, customize) - return walk + for g in deparsed.mod_globs: + deparsed.write('# global %s ## Warning: Unused global' % g) + + return deparsed if __name__ == '__main__': def deparse_test(co): + "This is a docstring" sys_version = sys.version_info.major + (sys.version_info.minor / 10.0) deparsed = deparse_code(sys_version, co, showasm=False, showast=False) + # deparsed = deparse_code(sys_version, co, showasm=False, showast=False, + # showgrammar=True) print(deparsed.text) return deparse_test(deparse_test.__code__)