From 9c5addc0f0cf10a75ef7100e4f61adcd7277b580 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 4 Feb 2024 13:56:28 -0500 Subject: [PATCH] Python 2.5 compatability --- uncompyle6/main.py | 45 +--------- uncompyle6/scanner.py | 11 ++- uncompyle6/scanners/scanner15.py | 11 +-- uncompyle6/scanners/scanner26.py | 128 +++++++++++++++++---------- uncompyle6/scanners/scanner27.py | 12 +-- uncompyle6/scanners/scanner3.py | 23 +---- uncompyle6/scanners/scanner37base.py | 2 +- uncompyle6/scanners/scanner38.py | 18 ++-- uncompyle6/semantics/fragments.py | 24 +++-- uncompyle6/semantics/n_actions.py | 17 ++-- uncompyle6/semantics/pysource.py | 3 +- uncompyle6/util.py | 9 +- 12 files changed, 133 insertions(+), 170 deletions(-) diff --git a/uncompyle6/main.py b/uncompyle6/main.py index d92a2ad7..020d43fa 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -307,17 +307,6 @@ def main( outstream = sys.stdout if do_linemaps: linemap_stream = sys.stdout - if do_verify: - prefix = os.path.basename(filename) + "-" - if prefix.endswith(".py"): - prefix = prefix[: -len(".py")] - - # Unbuffer output if possible - if sys.stdout.isatty(): - buffering = -1 - else: - buffering = 0 - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering) else: if filename.endswith(".pyc"): current_outfile = os.path.join(out_base, filename[0:-1]) @@ -399,39 +388,7 @@ def main( else: # uncompile successful if current_outfile: outstream.close() - if do_verify: - try: - msg = verify.compare_code_with_srcfile( - infile, current_outfile, do_verify - ) - if not current_outfile: - if not msg: - print("\n# okay decompiling %s" % infile) - okay_files += 1 - else: - verify_failed_files += 1 - print("\n# %s\n\t%s", infile, msg) - pass - else: - okay_files += 1 - pass - except verify.VerifyCmpError, e: - print(e) - verify_failed_files += 1 - os.rename(current_outfile, current_outfile + "_unverified") - sys.stderr.write("### Error Verifying %s\n" % filename) - sys.stderr.write(str(e) + "\n") - if not outfile: - sys.stderr.write("### Error Verifiying %s" % - filename) - sys.stderr.write(e) - if raise_on_error: - raise - pass - pass - pass - else: - okay_files += 1 + okay_files += 1 pass elif do_verify: sys.stderr.write("\n### uncompile successful, " diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index cb822f02..c04039d8 100644 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -32,9 +32,7 @@ from xdis import ( instruction_size, next_offset, ) -from xdis.version_info import PYTHON_VERSION_TRIPLE - -from xdis.version_info import IS_PYPY, version_tuple_to_str +from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str from uncompyle6.scanners.tok import Token @@ -106,7 +104,7 @@ class Code(object): class Scanner: - def __init__(self, version: tuple, show_asm=None, is_pypy=False): + def __init__(self, version, show_asm=None, is_pypy=False): self.version = version self.show_asm = show_asm self.is_pypy = is_pypy @@ -321,7 +319,7 @@ class Scanner: return arg def next_offset(self, op, offset): - return xdis.next_offset(op, self.opc, offset) + return next_offset(op, self.opc, offset) def print_bytecode(self): for i in self.op_range(0, len(self.code)): @@ -646,7 +644,7 @@ def get_scanner(version, is_pypy=False, show_asm=None): raise RuntimeError( "Import Python version, %s, for decompilation failed" % version_tuple_to_str(version) - ) + ) if is_pypy: scanner_class_name = "ScannerPyPy%s" % v_str @@ -671,5 +669,6 @@ if __name__ == "__main__": # scanner = get_scanner('2.7.13', True) # scanner = get_scanner(sys.version[:5], False) from xdis.version_info import PYTHON_VERSION_TRIPLE + scanner = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY, True) tokens, customize = scanner.ingest(co, {}, show_asm="after") diff --git a/uncompyle6/scanners/scanner15.py b/uncompyle6/scanners/scanner15.py index 6f187678..6a894a2a 100644 --- a/uncompyle6/scanners/scanner15.py +++ b/uncompyle6/scanners/scanner15.py @@ -6,21 +6,18 @@ This massages tokenized 1.5 bytecode to make it more amenable for grammar parsing. """ +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_15 + import uncompyle6.scanners.scanner21 as scan # from uncompyle6.scanners.scanner26 import ingest as ingest26 -# bytecode verification, verify(), uses JUMP_OPs from here -from xdis.opcodes import opcode_15 JUMP_OPS = opcode_15.JUMP_OPS -<<<<<<< HEAD -# We base this off of 2.1 instead of the other way around -======= -# We base this off of 2.2 instead of the other way around ->>>>>>> python-3.0-to-3.2 +# We base this off of 2.1 instead of the other way around # because we cleaned things up this way. # The history is that 2.7 support is the cleanest, # then from that we got 2.6 and so on. diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index b13ae49d..a8eaef86 100755 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -22,25 +22,28 @@ other versions of Python. Also, we save token information for later use in deparsing. """ -import uncompyle6.scanners.scanner2 as scan - -# bytecode verification, verify(), uses JUMP_OPs from here +# bytecode verification, verify(), uses jump_ops from here from xdis import iscode -from xdis.opcodes import opcode_26 from xdis.bytecode import _get_const_info +from xdis.opcodes import opcode_26 from uncompyle6.scanner import Token +from uncompyle6.scanners.scanner2 import Scanner2 JUMP_OPS = opcode_26.JUMP_OPS -class Scanner26(scan.Scanner2): + +class Scanner26(Scanner2): def __init__(self, show_asm=False): - super(Scanner26, self).__init__((2, 6), show_asm) + Scanner2.__init__(self, (2, 6), show_asm) # "setup" opcodes - self.setup_ops = frozenset([ - self.opc.SETUP_EXCEPT, self.opc.SETUP_FINALLY, - ]) + self.setup_ops = frozenset( + [ + self.opc.SETUP_EXCEPT, + self.opc.SETUP_FINALLY, + ] + ) return @@ -93,17 +96,18 @@ class Scanner26(scan.Scanner2): # 'LOAD_ASSERT' is used in assert statements. self.load_asserts = set() for i in self.op_range(0, codelen): - # We need to detect the difference between: # raise AssertionError # and # assert ... - if (self.code[i] == self.opc.JUMP_IF_TRUE and - i + 4 < codelen and - self.code[i+3] == self.opc.POP_TOP and - self.code[i+4] == self.opc.LOAD_GLOBAL): - if names[self.get_argument(i+4)] == 'AssertionError': - self.load_asserts.add(i+4) + if ( + self.code[i] == self.opc.JUMP_IF_TRUE + and i + 4 < codelen + and self.code[i + 3] == self.opc.POP_TOP + and self.code[i + 4] == self.opc.LOAD_GLOBAL + ): + if names[self.get_argument(i + 4)] == "AssertionError": + self.load_asserts.add(i + 4) jump_targets = self.find_jump_targets(show_asm) # contains (code, [addrRefToCode]) @@ -128,7 +132,8 @@ class Scanner26(scan.Scanner2): i += 1 op = self.code[offset] op_name = self.opname[op] - oparg = None; pattr = None + oparg = None + pattr = None if offset in jump_targets: jump_idx = 0 @@ -139,28 +144,37 @@ class Scanner26(scan.Scanner2): # properly. For example, a "loop" with an "if" nested in it should have the # "loop" tag last so the grammar rule matches that properly. last_jump_offset = -1 - for jump_offset in sorted(jump_targets[offset], reverse=True): + for jump_offset in sorted(jump_targets[offset], reverse=True): if jump_offset != last_jump_offset: - tokens.append(Token( - 'COME_FROM', jump_offset, repr(jump_offset), - offset="%s_%d" % (offset, jump_idx), - has_arg = True)) + tokens.append( + Token( + "COME_FROM", + jump_offset, + repr(jump_offset), + offset="%s_%d" % (offset, jump_idx), + has_arg=True, + ) + ) jump_idx += 1 last_jump_offset = jump_offset elif offset in self.thens: - tokens.append(Token( - 'THEN', None, self.thens[offset], - offset="%s_0" % offset, - has_arg = True)) + tokens.append( + Token( + "THEN", + None, + self.thens[offset], + offset="%s_0" % offset, + has_arg=True, + ) + ) - has_arg = (op >= self.opc.HAVE_ARGUMENT) + has_arg = op >= self.opc.HAVE_ARGUMENT if has_arg: oparg = self.get_argument(offset) + extended_arg extended_arg = 0 if op == self.opc.EXTENDED_ARG: - extended_arg += self.extended_arg_val(oparg) - continue - + extended_arg += self.extended_arg_val(oparg) + continue # Note: name used to match on rather than op since # BUILD_SET isn't in earlier Pythons. @@ -169,7 +183,14 @@ class Scanner26(scan.Scanner2): "BUILD_SET", ): t = Token( - op_name, oparg, pattr, offset, self.linestarts.get(offset, None), op, has_arg, self.opc + op_name, + oparg, + pattr, + offset, + self.linestarts.get(offset, None), + op, + has_arg, + self.opc, ) collection_type = op_name.split("_")[1] @@ -218,8 +239,8 @@ class Scanner26(scan.Scanner2): # FIXME: this is a hack to catch stuff like: # if x: continue # the "continue" is not on a new line. - if len(tokens) and tokens[-1].kind == 'JUMP_BACK': - tokens[-1].kind = intern('CONTINUE') + if len(tokens) and tokens[-1].kind == "JUMP_BACK": + tokens[-1].kind = intern("CONTINUE") elif op in self.opc.JABS_OPS: pattr = repr(oparg) @@ -237,17 +258,23 @@ class Scanner26(scan.Scanner2): # CE - Hack for >= 2.5 # Now all values loaded via LOAD_CLOSURE are packed into # a tuple before calling MAKE_CLOSURE. - if (self.version >= (2, 5) and op == self.opc.BUILD_TUPLE and - self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE): + if ( + self.version >= (2, 5) + and op == self.opc.BUILD_TUPLE + and self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE + ): continue else: - op_name = '%s_%d' % (op_name, oparg) + op_name = "%s_%d" % (op_name, oparg) customize[op_name] = oparg elif self.version > (2, 0) and op == self.opc.CONTINUE_LOOP: customize[op_name] = 0 - elif op_name in """ + elif ( + op_name + in """ CONTINUE_LOOP EXEC_STMT LOAD_LISTCOMP LOAD_SETCOMP - """.split(): + """.split() + ): customize[op_name] = 0 elif op == self.opc.JUMP_ABSOLUTE: # Further classify JUMP_ABSOLUTE into backward jumps @@ -263,23 +290,24 @@ class Scanner26(scan.Scanner2): # rule for that. target = self.get_target(offset) if target <= offset: - op_name = 'JUMP_BACK' - if (offset in self.stmts - and self.code[offset+3] not in (self.opc.END_FINALLY, - self.opc.POP_BLOCK)): - if ((offset in self.linestarts and - tokens[-1].kind == 'JUMP_BACK') - or offset not in self.not_continue): - op_name = 'CONTINUE' + op_name = "JUMP_BACK" + if offset in self.stmts and self.code[offset + 3] not in ( + self.opc.END_FINALLY, + self.opc.POP_BLOCK, + ): + if ( + offset in self.linestarts and tokens[-1].kind == "JUMP_BACK" + ) or offset not in self.not_continue: + op_name = "CONTINUE" else: # FIXME: this is a hack to catch stuff like: # if x: continue # the "continue" is not on a new line. - if tokens[-1].kind == 'JUMP_BACK': + if tokens[-1].kind == "JUMP_BACK": # We need 'intern' since we have # already have processed the previous # token. - tokens[-1].kind = intern('CONTINUE') + tokens[-1].kind = intern("CONTINUE") elif op == self.opc.LOAD_GLOBAL: if offset in self.load_asserts: @@ -331,4 +359,6 @@ if __name__ == "__main__": print(t.format()) pass else: - print("Need to be Python 2.6 to demo; I am version %s." % version_tuple_to_str()) + print( + "Need to be Python 2.6 to demo; I am version %s." % version_tuple_to_str() + ) diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index 01f4ee5a..ddb8e8ee 100755 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -7,20 +7,20 @@ grammar parsing. """ -from uncompyle6.scanners.scanner2 import Scanner2 - -from xdis.version_info import version_tuple_to_str import sys # bytecode verification, verify(), uses JUMP_OPs from here from xdis.opcodes import opcode_27 +from xdis.version_info import version_tuple_to_str + +from uncompyle6.scanners.scanner2 import Scanner2 JUMP_OPS = opcode_27.JUMP_OPs class Scanner27(Scanner2): def __init__(self, show_asm=False, is_pypy=False): - super(Scanner27, self).__init__((2, 7), show_asm, is_pypy) + Scanner2.__init__(self, (2, 7), show_asm, is_pypy) # opcodes that start statements self.statement_opcodes = frozenset( @@ -117,4 +117,6 @@ if __name__ == "__main__": print(t.format()) pass else: - print("Need to be Python 2.7 to demo; I am version %s." % version_tuple_to_str()) + print( + "Need to be Python 2.7 to demo; I am version %s." % version_tuple_to_str() + ) diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 7f2486c5..b3d520da 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -33,11 +33,6 @@ For example: Finally we save token information. """ -from xdis import iscode, instruction_size -from xdis.bytecode import _get_const_info - -from uncompyle6.scanners.tok import Token -from uncompyle6.scanner import parse_fn_counts_30_35 import xdis # Get all the opcodes into globals @@ -54,7 +49,7 @@ globals().update(op3.opmap) class Scanner3(Scanner): def __init__(self, version, show_asm=None, is_pypy=False): - super(Scanner3, self).__init__(version, show_asm, is_pypy) + Scanner.__init__(self, version, show_asm, is_pypy) # Create opcode classification sets # Note: super initialization above initializes self.opc @@ -286,8 +281,7 @@ class Scanner3(Scanner): ) return new_tokens - def bound_map_from_inst( - self, insts, next_tokens, inst, t, i): + def bound_map_from_inst(self, insts, next_tokens, inst, t, i): """ Try to a sequence of instruction that ends with a BUILD_MAP into a sequence that can be parsed much faster, but inserting the @@ -1532,16 +1526,3 @@ class Scanner3(Scanner): instr_offsets = filtered filtered = [] return instr_offsets - - -if __name__ == "__main__": - import inspect - - from xdis.version_info import PYTHON_VERSION_TRIPLE - - co = inspect.currentframe().f_code - - tokens, customize = Scanner3(PYTHON_VERSION_TRIPLE).ingest(co) - for t in tokens: - print(t) ->>>>>>> python-3.0-to-3.2 diff --git a/uncompyle6/scanners/scanner37base.py b/uncompyle6/scanners/scanner37base.py index 2aa457e9..10a2d3e3 100644 --- a/uncompyle6/scanners/scanner37base.py +++ b/uncompyle6/scanners/scanner37base.py @@ -48,7 +48,7 @@ CONST_COLLECTIONS = ("CONST_LIST", "CONST_SET", "CONST_DICT") class Scanner37Base(Scanner): def __init__(self, version, show_asm=None, debug="", is_pypy=False): - super(Scanner37Base, self).__init__(version, show_asm, is_pypy) + Scanner.__init__(self, version, show_asm, is_pypy) self.offset2tok_index = None self.debug = debug self.is_pypy = is_pypy diff --git a/uncompyle6/scanners/scanner38.py b/uncompyle6/scanners/scanner38.py index 6fad6c77..16858328 100644 --- a/uncompyle6/scanners/scanner38.py +++ b/uncompyle6/scanners/scanner38.py @@ -22,13 +22,13 @@ This sets up opcodes Python's 3.8 and calls a generalized scanner routine for Python 3.7 and up. """ -from uncompyle6.scanners.tok import off2int -from uncompyle6.scanners.scanner37 import Scanner37 -from uncompyle6.scanners.scanner37base import Scanner37Base - # bytecode verification, verify(), uses JUMP_OPs from here from xdis.opcodes import opcode_38 as opc +from uncompyle6.scanners.scanner37 import Scanner37 +from uncompyle6.scanners.scanner37base import Scanner37Base +from uncompyle6.scanners.tok import off2int + # bytecode verification, verify(), uses JUMP_OPS from here JUMP_OPs = opc.JUMP_OPS @@ -60,8 +60,8 @@ class Scanner38(Scanner37): grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST cause specific rules for the specific number of arguments they take. """ - tokens, customize = super(Scanner38, self).ingest( - bytecode, classname, code_objects, show_asm + tokens, customize = Scanner37.ingest( + self, bytecode, classname, code_objects, show_asm ) # Hacky way to detect loop ranges. @@ -93,7 +93,7 @@ class Scanner38(Scanner37): if len(loop_ends): next_end = loop_ends[-1] else: - next_end = tokens[len(tokens)-1].off2int() + 10 + next_end = tokens[len(tokens) - 1].off2int() + 10 # things that smash new_tokens like BUILD_LIST have to come first. @@ -161,4 +161,6 @@ if __name__ == "__main__": print(t.format()) pass else: - print("Need to be Python 3.8 to demo; I am version %s." % version_tuple_to_str()) + print( + "Need to be Python 3.8 to demo; I am version %s." % version_tuple_to_str() + ) diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 9f1ff29a..a4f908a1 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -65,7 +65,6 @@ The node position 0 will be associated with "import". import re from bisect import bisect_right -from collections import namedtuple from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser.ast import GenericASTTraversalPruningException @@ -77,7 +76,7 @@ from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.scanner import Code, Token, get_scanner from uncompyle6.semantics import pysource from uncompyle6.semantics.check_ast import checker -from uncompyle6.semantics.pysource import ParserError +from uncompyle6.semantics.parser_error import ParserError from uncompyle6.show import maybe_show_asm, maybe_show_tree if PYTHON_VERSION_TRIPLE < (2, 5): @@ -94,19 +93,18 @@ from uncompyle6.semantics.consts import ( TABLE_DIRECT, escape, ) +from uncompyle6.semantics.pysource import ( + DEFAULT_DEBUG_OPTS, + TREE_DEFAULT_DEBUG, + StringIO, +) +from uncompyle6.show import maybe_show_asm, maybe_show_tree + if PYTHON_VERSION_TRIPLE < (2, 6): from xdis.namedtuple24 import namedtuple else: from collections import namedtuple -from uncompyle6.semantics.pysource import ( - DEFAULT_DEBUG_OPTS, - TREE_DEFAULT_DEBUG, - ParserError, - StringIO, -) -from uncompyle6.show import maybe_show_asm, maybe_show_tree - NodeInfo = namedtuple("NodeInfo", "node start finish") ExtractInfo = namedtuple( "ExtractInfo", @@ -166,7 +164,7 @@ class FragmentsWalker(pysource.SourceWalker, object): def __init__( self, - version: tuple, + version, scanner, showast=TREE_DEFAULT_DEBUG, debug_parser=PARSER_DEFAULT_DEBUG, @@ -1201,7 +1199,7 @@ class FragmentsWalker(pysource.SourceWalker, object): ast = python_parser.parse(self.p, tokens, customize, code) self.customize(customize) self.p.insts = p_insts - except (parser.ParserError(e), AssertionError(e)): + except (ParserError(e), AssertionError(e)): raise ParserError(e, tokens) transform_tree = self.treeTransform.transform(ast, code) maybe_show_tree(self, ast) @@ -1241,7 +1239,7 @@ class FragmentsWalker(pysource.SourceWalker, object): self.p.opc = self.scanner.opc ast = python_parser.parse(self.p, tokens, customize, code) self.p.insts = p_insts - except (parser.ParserError(e), AssertionError(e)): + except (ParserError(e), AssertionError(e)): raise ParserError(e, tokens, {}) checker(ast, False, self.ast_errors) diff --git a/uncompyle6/semantics/n_actions.py b/uncompyle6/semantics/n_actions.py index 0f9ef18e..f93ab0d4 100644 --- a/uncompyle6/semantics/n_actions.py +++ b/uncompyle6/semantics/n_actions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023 by Rocky Bernstein +# Copyright (c) 2022-2024 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,16 +18,10 @@ Custom Nonterminal action functions. See NonterminalActions docstring. from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.scanners.tok import Token - -from uncompyle6.semantics.helper import ( - find_code_node, - flatten_list, -) -from uncompyle6.util import better_repr - from uncompyle6.semantics.consts import INDENT_PER_LEVEL, NONE, PRECEDENCE, minint from uncompyle6.semantics.helper import find_code_node, flatten_list -from uncompyle6.util import better_repr, get_code_name +from uncompyle6.util import better_repr + class NonterminalActions: """ @@ -719,8 +713,9 @@ class NonterminalActions: iter_index = 4 else: iter_index = 3 - self.comprehension_walk(node, iter_index=iter_index, - code_index=code_index) + self.comprehension_walk( + node, iter_index=iter_index, code_index=code_index + ) pass pass else: diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 6b2133ff..244daa53 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -168,6 +168,7 @@ from uncompyle6.semantics.parser_error import ParserError from uncompyle6.semantics.transform import TreeTransform, is_docstring from uncompyle6.show import maybe_show_tree from uncompyle6.util import better_repr + if PYTHON_VERSION_TRIPLE < (2, 5): from cStringIO import StringIO else: @@ -1260,7 +1261,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): self.p.opc = self.scanner.opc ast = parse(self.p, tokens, customize, code) self.p.insts = p_insts - except python_parser.ParserError, e: + except ParserError, e: raise ParserError(e, tokens, self.p.debug["reduce"]) checker(ast, False, self.ast_errors) diff --git a/uncompyle6/util.py b/uncompyle6/util.py index 313e388a..186ba3e2 100644 --- a/uncompyle6/util.py +++ b/uncompyle6/util.py @@ -4,22 +4,23 @@ try: from math import copysign + def is_negative_zero(n): """Returns true if n is -0.0""" return n == 0.0 and copysign(1, n) == -1 + except Exception: + def is_negative_zero(n): return False -def get_code_name(code) -> str: + +def get_code_name(code): code_name = code.co_name if isinstance(code_name, UnicodeForPython3): return code_name.value.decode("utf-8") return code_name -def is_negative_zero(n): - """Returns true if n is -0.0""" - return n == 0.0 and copysign(1, n) == -1 def better_repr(v, version): """Work around Python's unorthogonal and unhelpful repr() for primitive float