diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index b11479e2..db395b02 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -13,7 +13,6 @@ import time import click from xdis.version_info import version_tuple_to_str -from uncompyle6 import verify from uncompyle6.main import main, status_msg from uncompyle6.version import __version__ @@ -159,7 +158,7 @@ def main_bin( version_tuple = sys.version_info[0:2] if not ((3, 3) <= version_tuple < (3, 6)): print( - "Error: This version of the {program} runs from Python 3.3 to 3.6." + "Error: This version of the {program} runs from Python 3.3 to 3.5." "You need another branch of this code for other Python versions." " \n\tYou have version: %s." % version_tuple_to_str() ) diff --git a/uncompyle6/main.py b/uncompyle6/main.py index e970c344..0465a1ec 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -13,10 +13,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import ast import datetime import os import os.path as osp import py_compile +import subprocess import sys import tempfile @@ -50,6 +52,17 @@ def _get_outstream(outfile): return open(outfile, mode="w", encoding="utf-8") +def syntax_check(filename: str) -> bool: + with open(filename) as f: + source = f.read() + valid = True + try: + ast.parse(source) + except SyntaxError: + valid = False + return valid + + def decompile( co, bytecode_version=PYTHON_VERSION_TRIPLE, @@ -369,15 +382,22 @@ def main( check_type = "syntax check" if do_verify == "run": check_type = "run" - result = subprocess.run( - [sys.executable, deparsed_object.f.name], - capture_output=True, - ) - valid = result.returncode == 0 - output = result.stdout.decode() - if output: - print(output) - pass + if PYTHON_VERSION_TRIPLE >= (3, 7): + result = subprocess.run( + [sys.executable, deparsed_object.f.name], + capture_output=True, + ) + valid = result.returncode == 0 + output = result.stdout.decode() + if output: + print(output) + pass + else: + result = subprocess.run( + [sys.executable, deparsed_object.f.name], + ) + valid = result.returncode == 0 + pass if not valid: print(result.stderr.decode()) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 813f6660..f25ae5d7 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -27,22 +27,24 @@ that a later phase can turn into a sequence of ASCII text. """ import re -from uncompyle6.scanners.tok import Token + +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG + from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parsers.reducecheck import ( and_invalid, except_handler_else, ifelsestmt, - ifstmt, iflaststmt, + ifstmt, or_check, testtrue, tryelsestmtl3, tryexcept, - while1stmt + while1stmt, ) from uncompyle6.parsers.treenode import SyntaxTree -from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.scanners.tok import Token class Python3Parser(PythonParser): @@ -98,7 +100,7 @@ class Python3Parser(PythonParser): """ def p_dict_comp3(self, args): - """" + """ " expr ::= dict_comp stmt ::= dict_comp_func dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store @@ -519,7 +521,7 @@ class Python3Parser(PythonParser): expr call CALL_FUNCTION_3 - """ + """ # FIXME: I bet this can be simplified # look for next MAKE_FUNCTION for i in range(i + 1, len(tokens)): @@ -625,7 +627,11 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, token.kind, uniq_param, customize) if "LOAD_BUILD_CLASS" in self.seen_ops: - if next_token == "CALL_FUNCTION" and next_token.attr == 1 and pos_args_count > 1: + if ( + next_token == "CALL_FUNCTION" + and next_token.attr == 1 + and pos_args_count > 1 + ): rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % ( ("expr " * (pos_args_count - 1)), opname, @@ -764,18 +770,24 @@ class Python3Parser(PythonParser): elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): if opname == "BUILD_CONST_DICT": - rule = """ + rule = ( + """ add_consts ::= ADD_VALUE* const_list ::= COLLECTION_START add_consts %s dict ::= const_list expr ::= dict - """ % opname + """ + % opname + ) else: - rule = """ + rule = ( + """ add_consts ::= ADD_VALUE* const_list ::= COLLECTION_START add_consts %s expr ::= const_list - """ % opname + """ + % opname + ) self.addRule(rule, nop_func) elif opname.startswith("BUILD_DICT_OLDER"): @@ -854,18 +866,24 @@ class Python3Parser(PythonParser): elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): if opname == "BUILD_CONST_DICT": - rule = """ + rule = ( + """ add_consts ::= ADD_VALUE* const_list ::= COLLECTION_START add_consts %s dict ::= const_list expr ::= dict - """ % opname + """ + % opname + ) else: - rule = """ + rule = ( + """ add_consts ::= ADD_VALUE* const_list ::= COLLECTION_START add_consts %s expr ::= const_list - """ % opname + """ + % opname + ) self.addRule(rule, nop_func) elif opname_base in ( @@ -946,7 +964,6 @@ class Python3Parser(PythonParser): "CALL_FUNCTION_VAR_KW", ) ) or opname.startswith("CALL_FUNCTION_KW"): - if opname == "CALL_FUNCTION" and token.attr == 1: rule = """ dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr @@ -1122,7 +1139,8 @@ class Python3Parser(PythonParser): if has_get_iter_call_function1: rule_pat = ( "generator_exp ::= %sload_closure load_genexpr %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) + "GET_ITER CALL_FUNCTION_1" + % ("pos_arg " * pos_args_count, opname) ) self.add_make_function_rule(rule_pat, opname, token.attr, customize) @@ -1190,6 +1208,8 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) elif (3, 3) <= self.version < (3, 6): + # FIXME move this into version-specific custom rules. + # In fact, some of this has been done for 3.3. if annotate_args > 0: rule = ( "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" @@ -1208,7 +1228,6 @@ class Python3Parser(PythonParser): ) self.add_unique_rule(rule, opname, token.attr, customize) - if self.version >= (3, 4): if not self.is_pypy: load_op = "LOAD_STR" @@ -1292,14 +1311,16 @@ class Python3Parser(PythonParser): if has_get_iter_call_function1: rule_pat = ( "generator_exp ::= %sload_genexpr %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) + "GET_ITER CALL_FUNCTION_1" + % ("pos_arg " * pos_args_count, opname) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize ) rule_pat = ( "generator_exp ::= %sload_closure load_genexpr %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) + "GET_ITER CALL_FUNCTION_1" + % ("pos_arg " * pos_args_count, opname) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize @@ -1351,7 +1372,8 @@ class Python3Parser(PythonParser): if has_get_iter_call_function1: rule_pat = ( "generator_exp ::= %sload_genexpr %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) + "GET_ITER CALL_FUNCTION_1" + % ("pos_arg " * pos_args_count, opname) ) self.add_make_function_rule(rule_pat, opname, token.attr, customize) @@ -1363,7 +1385,8 @@ class Python3Parser(PythonParser): # Todo: For Pypy we need to modify this slightly rule_pat = ( "listcomp ::= %sLOAD_LISTCOMP %%s%s expr " - "GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) + "GET_ITER CALL_FUNCTION_1" + % ("expr " * pos_args_count, opname) ) self.add_make_function_rule( rule_pat, opname, token.attr, customize @@ -1450,9 +1473,6 @@ class Python3Parser(PythonParser): ) ) if self.version >= (3, 3): - # Normally we remove EXTENDED_ARG from the opcodes, but in the case of - # annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function. - # Yes this is a little hacky if self.version == (3, 3): # 3.3 puts kwargs before pos_arg pos_kw_tuple = ( @@ -1466,17 +1486,17 @@ class Python3Parser(PythonParser): ("kwargs " * kw_args_count), ) rule = ( - "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s" + "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s" % ( pos_kw_tuple[0], pos_kw_tuple[1], - ("call " * annotate_args), + ("annotate_arg " * annotate_args), opname, ) ) self.add_unique_rule(rule, opname, token.attr, customize) rule = ( - "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s" + "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s" % ( pos_kw_tuple[0], pos_kw_tuple[1], @@ -1485,9 +1505,8 @@ class Python3Parser(PythonParser): ) ) else: - # See above comment about use of EXTENDED_ARG rule = ( - "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s" + "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s" % ( ("kwargs " * kw_args_count), ("pos_arg " * (pos_args_count)), @@ -1497,7 +1516,7 @@ class Python3Parser(PythonParser): ) self.add_unique_rule(rule, opname, token.attr, customize) rule = ( - "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s" + "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s" % ( ("kwargs " * kw_args_count), ("pos_arg " * pos_args_count), @@ -1594,7 +1613,7 @@ class Python3Parser(PythonParser): } if self.version == (3, 6): - self.reduce_check_table["and"] = and_invalid + self.reduce_check_table["and"] = and_invalid self.check_reduce["and"] = "AST" self.check_reduce["annotate_tuple"] = "noAST" @@ -1624,7 +1643,7 @@ class Python3Parser(PythonParser): def reduce_is_invalid(self, rule, ast, tokens, first, last): lhs = rule[0] n = len(tokens) - last = min(last, n-1) + last = min(last, n - 1) fn = self.reduce_check_table.get(lhs, None) if fn: if fn(self, lhs, n, rule, ast, tokens, first, last): @@ -1650,13 +1669,18 @@ class Python3Parser(PythonParser): condition_jump2 = tokens[min(last - 1, len(tokens) - 1)] # If there are two *distinct* condition jumps, they should not jump to the # same place. Otherwise we have some sort of "and"/"or". - if condition_jump2.kind.startswith("POP_JUMP_IF") and condition_jump != condition_jump2: + if ( + condition_jump2.kind.startswith("POP_JUMP_IF") + and condition_jump != condition_jump2 + ): return condition_jump.attr == condition_jump2.attr - if tokens[last] == "COME_FROM" and tokens[last].off2int() != condition_jump.attr: + if ( + tokens[last] == "COME_FROM" + and tokens[last].off2int() != condition_jump.attr + ): return False - # if condition_jump.attr < condition_jump2.off2int(): # print("XXX", first, last) # for t in range(first, last): print(tokens[t]) @@ -1678,7 +1702,6 @@ class Python3Parser(PythonParser): < tokens[last].off2int() ) elif lhs == "while1stmt": - if while1stmt(self, lhs, n, rule, ast, tokens, first, last): return True @@ -1700,7 +1723,6 @@ class Python3Parser(PythonParser): return True return False elif lhs == "while1elsestmt": - n = len(tokens) if last == n: # Adjust for fuzziness in parsing diff --git a/uncompyle6/parsers/parse33.py b/uncompyle6/parsers/parse33.py index 55432e72..ce1fc672 100644 --- a/uncompyle6/parsers/parse33.py +++ b/uncompyle6/parsers/parse33.py @@ -1,15 +1,13 @@ -# Copyright (c) 2016 Rocky Bernstein +# Copyright (c) 2016, 2024 Rocky Bernstein """ spark grammar differences over Python 3.2 for Python 3.3. """ -from __future__ import print_function from uncompyle6.parser import PythonParserSingle from uncompyle6.parsers.parse32 import Python32Parser class Python33Parser(Python32Parser): - def p_33on(self, args): """ # Python 3.3+ adds yield from. @@ -19,13 +17,22 @@ class Python33Parser(Python32Parser): """ def customize_grammar_rules(self, tokens, customize): - self.remove_rules(""" + self.remove_rules( + """ # 3.3+ adds POP_BLOCKS whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP - """) + """ + ) super(Python33Parser, self).customize_grammar_rules(tokens, customize) + + # FIXME: move 3.3 stuff out of parse3.py and put it here. + # for i, token in enumerate(tokens): + # opname = token.kind + # opname_base = opname[: opname.rfind("_")] + return + class Python33ParserSingle(Python33Parser, PythonParserSingle): pass diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 3491a25c..d997a40e 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -36,7 +36,6 @@ Finally we save token information. from __future__ import print_function import sys - import xdis # Get all the opcodes into globals @@ -479,6 +478,7 @@ class Scanner3(Scanner): last_op_was_break = False new_tokens = [] + operand_value = 0 for i, inst in enumerate(self.insts): opname = inst.opname @@ -530,10 +530,11 @@ class Scanner3(Scanner): op = inst.opcode if opname == "EXTENDED_ARG": - # FIXME: The EXTENDED_ARG is used to signal annotation - # parameters - if i + 1 < n and self.insts[i + 1].opcode != self.opc.MAKE_FUNCTION: + if i + 1 < n: + operand_value = argval << 16 continue + else: + operand_value = 0 if inst.offset in jump_targets: jump_idx = 0 @@ -640,7 +641,7 @@ class Scanner3(Scanner): attr = attr[:4] # remove last value: attr[5] == False else: pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35( - inst.argval + inst.argval + operand_value ) pattr = "%s positional, %s keyword only, %s annotated" % ( diff --git a/uncompyle6/scanners/scanner33.py b/uncompyle6/scanners/scanner33.py index 1d5d7503..1c4a5aa9 100644 --- a/uncompyle6/scanners/scanner33.py +++ b/uncompyle6/scanners/scanner33.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein +# Copyright (c) 2015-2019, 2021-2022, 2024 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,20 +19,22 @@ This sets up opcodes Python's 3.3 and calls a generalized scanner routine for Python 3. """ -from __future__ import print_function - # bytecode verification, verify(), uses JUMP_OPs from here from xdis.opcodes import opcode_33 as opc -JUMP_OPS = opc.JUMP_OPS from uncompyle6.scanners.scanner3 import Scanner3 -class Scanner33(Scanner3): +JUMP_OPS = opc.JUMP_OPS + + +class Scanner33(Scanner3): def __init__(self, show_asm=False, is_pypy=False): Scanner3.__init__(self, (3, 3), show_asm) return + pass + if __name__ == "__main__": from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str