jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 00:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-3.3-to-3.5

Browse Source
This commit is contained in:
rocky
2024-02-11 11:57:12 -05:00
parent ece788e09e 454fac4adb
commit c048b26d4e
6 changed files with 115 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
uncompyle6/bin/uncompile.py
View File

@@ -13,7 +13,6 @@ import time
import click import click
from xdis.version_info import version_tuple_to_str from xdis.version_info import version_tuple_to_str
from uncompyle6 import verify
from uncompyle6.main import main, status_msg from uncompyle6.main import main, status_msg
from uncompyle6.version import __version__ from uncompyle6.version import __version__
@@ -159,7 +158,7 @@ def main_bin(
version_tuple = sys.version_info[0:2] version_tuple = sys.version_info[0:2]
if not ((3, 3) <= version_tuple < (3, 6)): if not ((3, 3) <= version_tuple < (3, 6)):
print( print(
"Error: This version of the {program} runs from Python 3.3 to 3.6." "Error: This version of the {program} runs from Python 3.3 to 3.5."
"You need another branch of this code for other Python versions." "You need another branch of this code for other Python versions."
" \n\tYou have version: %s." % version_tuple_to_str() " \n\tYou have version: %s." % version_tuple_to_str()
) )

38
uncompyle6/main.py
View File

@@ -13,10 +13,12 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import ast
import datetime import datetime
import os import os
import os.path as osp import os.path as osp
import py_compile import py_compile
import subprocess
import sys import sys
import tempfile import tempfile
@@ -50,6 +52,17 @@ def _get_outstream(outfile):
return open(outfile, mode="w", encoding="utf-8") return open(outfile, mode="w", encoding="utf-8")
def syntax_check(filename: str) -> bool:
with open(filename) as f:
source = f.read()
valid = True
try:
ast.parse(source)
except SyntaxError:
valid = False
return valid
def decompile( def decompile(
co, co,
bytecode_version=PYTHON_VERSION_TRIPLE, bytecode_version=PYTHON_VERSION_TRIPLE,
@@ -369,15 +382,22 @@ def main(
check_type = "syntax check" check_type = "syntax check"
if do_verify == "run": if do_verify == "run":
check_type = "run" check_type = "run"
result = subprocess.run( if PYTHON_VERSION_TRIPLE >= (3, 7):
[sys.executable, deparsed_object.f.name], result = subprocess.run(
capture_output=True, [sys.executable, deparsed_object.f.name],
) capture_output=True,
valid = result.returncode == 0 )
output = result.stdout.decode() valid = result.returncode == 0
if output: output = result.stdout.decode()
print(output) if output:
pass print(output)
pass
else:
result = subprocess.run(
[sys.executable, deparsed_object.f.name],
)
valid = result.returncode == 0
pass
if not valid: if not valid:
print(result.stderr.decode()) print(result.stderr.decode())

98
uncompyle6/parsers/parse3.py
View File

@@ -27,22 +27,24 @@ that a later phase can turn into a sequence of ASCII text.
""" """
import re import re
from uncompyle6.scanners.tok import Token
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.reducecheck import ( from uncompyle6.parsers.reducecheck import (
and_invalid, and_invalid,
except_handler_else, except_handler_else,
ifelsestmt, ifelsestmt,
ifstmt,
iflaststmt, iflaststmt,
ifstmt,
or_check, or_check,
testtrue, testtrue,
tryelsestmtl3, tryelsestmtl3,
tryexcept, tryexcept,
while1stmt while1stmt,
) )
from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from uncompyle6.scanners.tok import Token
class Python3Parser(PythonParser): class Python3Parser(PythonParser):
@@ -98,7 +100,7 @@ class Python3Parser(PythonParser):
""" """
def p_dict_comp3(self, args): def p_dict_comp3(self, args):
"""" """ "
expr ::= dict_comp expr ::= dict_comp
stmt ::= dict_comp_func stmt ::= dict_comp_func
dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store
@@ -519,7 +521,7 @@ class Python3Parser(PythonParser):
expr expr
call call
CALL_FUNCTION_3 CALL_FUNCTION_3
""" """
# FIXME: I bet this can be simplified # FIXME: I bet this can be simplified
# look for next MAKE_FUNCTION # look for next MAKE_FUNCTION
for i in range(i + 1, len(tokens)): for i in range(i + 1, len(tokens)):
@@ -625,7 +627,11 @@ class Python3Parser(PythonParser):
self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule(rule, token.kind, uniq_param, customize)
if "LOAD_BUILD_CLASS" in self.seen_ops: if "LOAD_BUILD_CLASS" in self.seen_ops:
if next_token == "CALL_FUNCTION" and next_token.attr == 1 and pos_args_count > 1: if (
next_token == "CALL_FUNCTION"
and next_token.attr == 1
and pos_args_count > 1
):
rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % ( rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
("expr " * (pos_args_count - 1)), ("expr " * (pos_args_count - 1)),
opname, opname,
@@ -764,18 +770,24 @@ class Python3Parser(PythonParser):
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
if opname == "BUILD_CONST_DICT": if opname == "BUILD_CONST_DICT":
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
dict ::= const_list dict ::= const_list
expr ::= dict expr ::= dict
""" % opname """
% opname
)
else: else:
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
expr ::= const_list expr ::= const_list
""" % opname """
% opname
)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname.startswith("BUILD_DICT_OLDER"): elif opname.startswith("BUILD_DICT_OLDER"):
@@ -854,18 +866,24 @@ class Python3Parser(PythonParser):
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
if opname == "BUILD_CONST_DICT": if opname == "BUILD_CONST_DICT":
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
dict ::= const_list dict ::= const_list
expr ::= dict expr ::= dict
""" % opname """
% opname
)
else: else:
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
expr ::= const_list expr ::= const_list
""" % opname """
% opname
)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname_base in ( elif opname_base in (
@@ -946,7 +964,6 @@ class Python3Parser(PythonParser):
"CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_VAR_KW",
) )
) or opname.startswith("CALL_FUNCTION_KW"): ) or opname.startswith("CALL_FUNCTION_KW"):
if opname == "CALL_FUNCTION" and token.attr == 1: if opname == "CALL_FUNCTION" and token.attr == 1:
rule = """ rule = """
dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
@@ -1122,7 +1139,8 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1: if has_get_iter_call_function1:
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_closure load_genexpr %%s%s expr " "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule(rule_pat, opname, token.attr, customize) self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@@ -1190,6 +1208,8 @@ class Python3Parser(PythonParser):
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
elif (3, 3) <= self.version < (3, 6): elif (3, 3) <= self.version < (3, 6):
# FIXME move this into version-specific custom rules.
# In fact, some of this has been done for 3.3.
if annotate_args > 0: if annotate_args > 0:
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s"
@@ -1208,7 +1228,6 @@ class Python3Parser(PythonParser):
) )
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
if self.version >= (3, 4): if self.version >= (3, 4):
if not self.is_pypy: if not self.is_pypy:
load_op = "LOAD_STR" load_op = "LOAD_STR"
@@ -1292,14 +1311,16 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1: if has_get_iter_call_function1:
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_genexpr %%s%s expr " "generator_exp ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
) )
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_closure load_genexpr %%s%s expr " "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
@@ -1351,7 +1372,8 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1: if has_get_iter_call_function1:
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_genexpr %%s%s expr " "generator_exp ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule(rule_pat, opname, token.attr, customize) self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@@ -1363,7 +1385,8 @@ class Python3Parser(PythonParser):
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
@@ -1450,9 +1473,6 @@ class Python3Parser(PythonParser):
) )
) )
if self.version >= (3, 3): if self.version >= (3, 3):
# Normally we remove EXTENDED_ARG from the opcodes, but in the case of
# annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
# Yes this is a little hacky
if self.version == (3, 3): if self.version == (3, 3):
# 3.3 puts kwargs before pos_arg # 3.3 puts kwargs before pos_arg
pos_kw_tuple = ( pos_kw_tuple = (
@@ -1466,17 +1486,17 @@ class Python3Parser(PythonParser):
("kwargs " * kw_args_count), ("kwargs " * kw_args_count),
) )
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
% ( % (
pos_kw_tuple[0], pos_kw_tuple[0],
pos_kw_tuple[1], pos_kw_tuple[1],
("call " * annotate_args), ("annotate_arg " * annotate_args),
opname, opname,
) )
) )
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
% ( % (
pos_kw_tuple[0], pos_kw_tuple[0],
pos_kw_tuple[1], pos_kw_tuple[1],
@@ -1485,9 +1505,8 @@ class Python3Parser(PythonParser):
) )
) )
else: else:
# See above comment about use of EXTENDED_ARG
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
% ( % (
("kwargs " * kw_args_count), ("kwargs " * kw_args_count),
("pos_arg " * (pos_args_count)), ("pos_arg " * (pos_args_count)),
@@ -1497,7 +1516,7 @@ class Python3Parser(PythonParser):
) )
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
% ( % (
("kwargs " * kw_args_count), ("kwargs " * kw_args_count),
("pos_arg " * pos_args_count), ("pos_arg " * pos_args_count),
@@ -1594,7 +1613,7 @@ class Python3Parser(PythonParser):
} }
if self.version == (3, 6): if self.version == (3, 6):
self.reduce_check_table["and"] = and_invalid self.reduce_check_table["and"] = and_invalid
self.check_reduce["and"] = "AST" self.check_reduce["and"] = "AST"
self.check_reduce["annotate_tuple"] = "noAST" self.check_reduce["annotate_tuple"] = "noAST"
@@ -1624,7 +1643,7 @@ class Python3Parser(PythonParser):
def reduce_is_invalid(self, rule, ast, tokens, first, last): def reduce_is_invalid(self, rule, ast, tokens, first, last):
lhs = rule[0] lhs = rule[0]
n = len(tokens) n = len(tokens)
last = min(last, n-1) last = min(last, n - 1)
fn = self.reduce_check_table.get(lhs, None) fn = self.reduce_check_table.get(lhs, None)
if fn: if fn:
if fn(self, lhs, n, rule, ast, tokens, first, last): if fn(self, lhs, n, rule, ast, tokens, first, last):
@@ -1650,13 +1669,18 @@ class Python3Parser(PythonParser):
condition_jump2 = tokens[min(last - 1, len(tokens) - 1)] condition_jump2 = tokens[min(last - 1, len(tokens) - 1)]
# If there are two *distinct* condition jumps, they should not jump to the # If there are two *distinct* condition jumps, they should not jump to the
# same place. Otherwise we have some sort of "and"/"or". # same place. Otherwise we have some sort of "and"/"or".
if condition_jump2.kind.startswith("POP_JUMP_IF") and condition_jump != condition_jump2: if (
condition_jump2.kind.startswith("POP_JUMP_IF")
and condition_jump != condition_jump2
):
return condition_jump.attr == condition_jump2.attr return condition_jump.attr == condition_jump2.attr
if tokens[last] == "COME_FROM" and tokens[last].off2int() != condition_jump.attr: if (
tokens[last] == "COME_FROM"
and tokens[last].off2int() != condition_jump.attr
):
return False return False
# if condition_jump.attr < condition_jump2.off2int(): # if condition_jump.attr < condition_jump2.off2int():
# print("XXX", first, last) # print("XXX", first, last)
# for t in range(first, last): print(tokens[t]) # for t in range(first, last): print(tokens[t])
@@ -1678,7 +1702,6 @@ class Python3Parser(PythonParser):
< tokens[last].off2int() < tokens[last].off2int()
) )
elif lhs == "while1stmt": elif lhs == "while1stmt":
if while1stmt(self, lhs, n, rule, ast, tokens, first, last): if while1stmt(self, lhs, n, rule, ast, tokens, first, last):
return True return True
@@ -1700,7 +1723,6 @@ class Python3Parser(PythonParser):
return True return True
return False return False
elif lhs == "while1elsestmt": elif lhs == "while1elsestmt":
n = len(tokens) n = len(tokens)
if last == n: if last == n:
# Adjust for fuzziness in parsing # Adjust for fuzziness in parsing

17
uncompyle6/parsers/parse33.py
View File

@@ -1,15 +1,13 @@
# Copyright (c) 2016 Rocky Bernstein # Copyright (c) 2016, 2024 Rocky Bernstein
""" """
spark grammar differences over Python 3.2 for Python 3.3. spark grammar differences over Python 3.2 for Python 3.3.
""" """
from __future__ import print_function
from uncompyle6.parser import PythonParserSingle from uncompyle6.parser import PythonParserSingle
from uncompyle6.parsers.parse32 import Python32Parser from uncompyle6.parsers.parse32 import Python32Parser
class Python33Parser(Python32Parser): class Python33Parser(Python32Parser):
def p_33on(self, args): def p_33on(self, args):
""" """
# Python 3.3+ adds yield from. # Python 3.3+ adds yield from.
@@ -19,13 +17,22 @@ class Python33Parser(Python32Parser):
""" """
def customize_grammar_rules(self, tokens, customize): def customize_grammar_rules(self, tokens, customize):
self.remove_rules(""" self.remove_rules(
"""
# 3.3+ adds POP_BLOCKS # 3.3+ adds POP_BLOCKS
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP
""") """
)
super(Python33Parser, self).customize_grammar_rules(tokens, customize) super(Python33Parser, self).customize_grammar_rules(tokens, customize)
# FIXME: move 3.3 stuff out of parse3.py and put it here.
# for i, token in enumerate(tokens):
# opname = token.kind
# opname_base = opname[: opname.rfind("_")]
return return
class Python33ParserSingle(Python33Parser, PythonParserSingle): class Python33ParserSingle(Python33Parser, PythonParserSingle):
pass pass

11
uncompyle6/scanners/scanner3.py
View File

@@ -36,7 +36,6 @@ Finally we save token information.
from __future__ import print_function from __future__ import print_function
import sys import sys
import xdis import xdis
# Get all the opcodes into globals # Get all the opcodes into globals
@@ -479,6 +478,7 @@ class Scanner3(Scanner):
last_op_was_break = False last_op_was_break = False
new_tokens = [] new_tokens = []
operand_value = 0
for i, inst in enumerate(self.insts): for i, inst in enumerate(self.insts):
opname = inst.opname opname = inst.opname
@@ -530,10 +530,11 @@ class Scanner3(Scanner):
op = inst.opcode op = inst.opcode
if opname == "EXTENDED_ARG": if opname == "EXTENDED_ARG":
# FIXME: The EXTENDED_ARG is used to signal annotation if i + 1 < n:
# parameters operand_value = argval << 16
if i + 1 < n and self.insts[i + 1].opcode != self.opc.MAKE_FUNCTION:
continue continue
else:
operand_value = 0
if inst.offset in jump_targets: if inst.offset in jump_targets:
jump_idx = 0 jump_idx = 0
@@ -640,7 +641,7 @@ class Scanner3(Scanner):
attr = attr[:4] # remove last value: attr[5] == False attr = attr[:4] # remove last value: attr[5] == False
else: else:
pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35( pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35(
inst.argval inst.argval + operand_value
) )
pattr = "%s positional, %s keyword only, %s annotated" % ( pattr = "%s positional, %s keyword only, %s annotated" % (

12
uncompyle6/scanners/scanner33.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein # Copyright (c) 2015-2019, 2021-2022, 2024 by Rocky Bernstein
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@@ -19,20 +19,22 @@ This sets up opcodes Python's 3.3 and calls a generalized
scanner routine for Python 3. scanner routine for Python 3.
""" """
from __future__ import print_function
# bytecode verification, verify(), uses JUMP_OPs from here # bytecode verification, verify(), uses JUMP_OPs from here
from xdis.opcodes import opcode_33 as opc from xdis.opcodes import opcode_33 as opc
JUMP_OPS = opc.JUMP_OPS
from uncompyle6.scanners.scanner3 import Scanner3 from uncompyle6.scanners.scanner3 import Scanner3
class Scanner33(Scanner3):
JUMP_OPS = opc.JUMP_OPS
class Scanner33(Scanner3):
def __init__(self, show_asm=False, is_pypy=False): def __init__(self, show_asm=False, is_pypy=False):
Scanner3.__init__(self, (3, 3), show_asm) Scanner3.__init__(self, (3, 3), show_asm)
return return
pass pass
if __name__ == "__main__": if __name__ == "__main__":
from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str