Handle 3.3 MAKE_FUNCTION annotation args properly

This commit is contained in:
rocky
2024-02-11 11:50:25 -05:00
parent 147155e1d5
commit 454fac4adb
6 changed files with 123 additions and 73 deletions

View File

@@ -159,10 +159,10 @@ def main_bin(
""" """
version_tuple = sys.version_info[0:2] version_tuple = sys.version_info[0:2]
if version_tuple < (3, 7): if version_tuple < (3, 6):
print( print(
f"Error: This version of the {program} runs from Python 3.7 or greater." f"Error: This version of the {program} runs from Python 3.6 or greater."
f"You need another branch of this code for Python before 3.7." f"You need another branch of this code for Python before 3.6."
f""" \n\tYou have version: {version_tuple_to_str()}.""" f""" \n\tYou have version: {version_tuple_to_str()}."""
) )
sys.exit(-1) sys.exit(-1)

View File

@@ -13,10 +13,12 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import ast
import datetime import datetime
import os import os
import os.path as osp import os.path as osp
import py_compile import py_compile
import subprocess
import sys import sys
import tempfile import tempfile
from typing import Any, Optional, TextIO, Tuple from typing import Any, Optional, TextIO, Tuple
@@ -51,6 +53,17 @@ def _get_outstream(outfile: str) -> Any:
return open(outfile, mode="w", encoding="utf-8") return open(outfile, mode="w", encoding="utf-8")
def syntax_check(filename: str) -> bool:
with open(filename) as f:
source = f.read()
valid = True
try:
ast.parse(source)
except SyntaxError:
valid = False
return valid
def decompile( def decompile(
co, co,
bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE, bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE,
@@ -368,15 +381,22 @@ def main(
check_type = "syntax check" check_type = "syntax check"
if do_verify == "run": if do_verify == "run":
check_type = "run" check_type = "run"
result = subprocess.run( if PYTHON_VERSION_TRIPLE >= (3, 7):
[sys.executable, deparsed_object.f.name], result = subprocess.run(
capture_output=True, [sys.executable, deparsed_object.f.name],
) capture_output=True,
valid = result.returncode == 0 )
output = result.stdout.decode() valid = result.returncode == 0
if output: output = result.stdout.decode()
print(output) if output:
pass print(output)
pass
else:
result = subprocess.run(
[sys.executable, deparsed_object.f.name],
)
valid = result.returncode == 0
pass
if not valid: if not valid:
print(result.stderr.decode()) print(result.stderr.decode())

View File

@@ -27,22 +27,24 @@ that a later phase can turn into a sequence of ASCII text.
""" """
import re import re
from uncompyle6.scanners.tok import Token
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.reducecheck import ( from uncompyle6.parsers.reducecheck import (
and_invalid, and_invalid,
except_handler_else, except_handler_else,
ifelsestmt, ifelsestmt,
ifstmt,
iflaststmt, iflaststmt,
ifstmt,
or_check, or_check,
testtrue, testtrue,
tryelsestmtl3, tryelsestmtl3,
tryexcept, tryexcept,
while1stmt while1stmt,
) )
from uncompyle6.parsers.treenode import SyntaxTree from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from uncompyle6.scanners.tok import Token
class Python3Parser(PythonParser): class Python3Parser(PythonParser):
@@ -98,7 +100,7 @@ class Python3Parser(PythonParser):
""" """
def p_dict_comp3(self, args): def p_dict_comp3(self, args):
"""" """ "
expr ::= dict_comp expr ::= dict_comp
stmt ::= dict_comp_func stmt ::= dict_comp_func
dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store
@@ -519,7 +521,7 @@ class Python3Parser(PythonParser):
expr expr
call call
CALL_FUNCTION_3 CALL_FUNCTION_3
""" """
# FIXME: I bet this can be simplified # FIXME: I bet this can be simplified
# look for next MAKE_FUNCTION # look for next MAKE_FUNCTION
for i in range(i + 1, len(tokens)): for i in range(i + 1, len(tokens)):
@@ -625,7 +627,11 @@ class Python3Parser(PythonParser):
self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule(rule, token.kind, uniq_param, customize)
if "LOAD_BUILD_CLASS" in self.seen_ops: if "LOAD_BUILD_CLASS" in self.seen_ops:
if next_token == "CALL_FUNCTION" and next_token.attr == 1 and pos_args_count > 1: if (
next_token == "CALL_FUNCTION"
and next_token.attr == 1
and pos_args_count > 1
):
rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % ( rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
("expr " * (pos_args_count - 1)), ("expr " * (pos_args_count - 1)),
opname, opname,
@@ -764,18 +770,24 @@ class Python3Parser(PythonParser):
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
if opname == "BUILD_CONST_DICT": if opname == "BUILD_CONST_DICT":
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
dict ::= const_list dict ::= const_list
expr ::= dict expr ::= dict
""" % opname """
% opname
)
else: else:
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
expr ::= const_list expr ::= const_list
""" % opname """
% opname
)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname.startswith("BUILD_DICT_OLDER"): elif opname.startswith("BUILD_DICT_OLDER"):
@@ -854,18 +866,24 @@ class Python3Parser(PythonParser):
elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"): elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
if opname == "BUILD_CONST_DICT": if opname == "BUILD_CONST_DICT":
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
dict ::= const_list dict ::= const_list
expr ::= dict expr ::= dict
""" % opname """
% opname
)
else: else:
rule = """ rule = (
"""
add_consts ::= ADD_VALUE* add_consts ::= ADD_VALUE*
const_list ::= COLLECTION_START add_consts %s const_list ::= COLLECTION_START add_consts %s
expr ::= const_list expr ::= const_list
""" % opname """
% opname
)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname_base in ( elif opname_base in (
@@ -946,7 +964,6 @@ class Python3Parser(PythonParser):
"CALL_FUNCTION_VAR_KW", "CALL_FUNCTION_VAR_KW",
) )
) or opname.startswith("CALL_FUNCTION_KW"): ) or opname.startswith("CALL_FUNCTION_KW"):
if opname == "CALL_FUNCTION" and token.attr == 1: if opname == "CALL_FUNCTION" and token.attr == 1:
rule = """ rule = """
dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr dict_comp ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
@@ -1122,7 +1139,8 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1: if has_get_iter_call_function1:
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_closure load_genexpr %%s%s expr " "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule(rule_pat, opname, token.attr, customize) self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@@ -1190,6 +1208,8 @@ class Python3Parser(PythonParser):
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
elif (3, 3) <= self.version < (3, 6): elif (3, 3) <= self.version < (3, 6):
# FIXME move this into version-specific custom rules.
# In fact, some of this has been done for 3.3.
if annotate_args > 0: if annotate_args > 0:
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s"
@@ -1208,7 +1228,6 @@ class Python3Parser(PythonParser):
) )
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
if self.version >= (3, 4): if self.version >= (3, 4):
if not self.is_pypy: if not self.is_pypy:
load_op = "LOAD_STR" load_op = "LOAD_STR"
@@ -1292,14 +1311,16 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1: if has_get_iter_call_function1:
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_genexpr %%s%s expr " "generator_exp ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
) )
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_closure load_genexpr %%s%s expr " "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
@@ -1351,7 +1372,8 @@ class Python3Parser(PythonParser):
if has_get_iter_call_function1: if has_get_iter_call_function1:
rule_pat = ( rule_pat = (
"generator_exp ::= %sload_genexpr %%s%s expr " "generator_exp ::= %sload_genexpr %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
) )
self.add_make_function_rule(rule_pat, opname, token.attr, customize) self.add_make_function_rule(rule_pat, opname, token.attr, customize)
@@ -1363,7 +1385,8 @@ class Python3Parser(PythonParser):
# Todo: For Pypy we need to modify this slightly # Todo: For Pypy we need to modify this slightly
rule_pat = ( rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr " "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname) "GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname)
) )
self.add_make_function_rule( self.add_make_function_rule(
rule_pat, opname, token.attr, customize rule_pat, opname, token.attr, customize
@@ -1450,9 +1473,6 @@ class Python3Parser(PythonParser):
) )
) )
if self.version >= (3, 3): if self.version >= (3, 3):
# Normally we remove EXTENDED_ARG from the opcodes, but in the case of
# annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
# Yes this is a little hacky
if self.version == (3, 3): if self.version == (3, 3):
# 3.3 puts kwargs before pos_arg # 3.3 puts kwargs before pos_arg
pos_kw_tuple = ( pos_kw_tuple = (
@@ -1466,17 +1486,17 @@ class Python3Parser(PythonParser):
("kwargs " * kw_args_count), ("kwargs " * kw_args_count),
) )
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
% ( % (
pos_kw_tuple[0], pos_kw_tuple[0],
pos_kw_tuple[1], pos_kw_tuple[1],
("call " * annotate_args), ("annotate_arg " * annotate_args),
opname, opname,
) )
) )
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR %s"
% ( % (
pos_kw_tuple[0], pos_kw_tuple[0],
pos_kw_tuple[1], pos_kw_tuple[1],
@@ -1485,9 +1505,8 @@ class Python3Parser(PythonParser):
) )
) )
else: else:
# See above comment about use of EXTENDED_ARG
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
% ( % (
("kwargs " * kw_args_count), ("kwargs " * kw_args_count),
("pos_arg " * (pos_args_count)), ("pos_arg " * (pos_args_count)),
@@ -1497,7 +1516,7 @@ class Python3Parser(PythonParser):
) )
self.add_unique_rule(rule, opname, token.attr, customize) self.add_unique_rule(rule, opname, token.attr, customize)
rule = ( rule = (
"mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s" "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
% ( % (
("kwargs " * kw_args_count), ("kwargs " * kw_args_count),
("pos_arg " * pos_args_count), ("pos_arg " * pos_args_count),
@@ -1594,7 +1613,7 @@ class Python3Parser(PythonParser):
} }
if self.version == (3, 6): if self.version == (3, 6):
self.reduce_check_table["and"] = and_invalid self.reduce_check_table["and"] = and_invalid
self.check_reduce["and"] = "AST" self.check_reduce["and"] = "AST"
self.check_reduce["annotate_tuple"] = "noAST" self.check_reduce["annotate_tuple"] = "noAST"
@@ -1624,7 +1643,7 @@ class Python3Parser(PythonParser):
def reduce_is_invalid(self, rule, ast, tokens, first, last): def reduce_is_invalid(self, rule, ast, tokens, first, last):
lhs = rule[0] lhs = rule[0]
n = len(tokens) n = len(tokens)
last = min(last, n-1) last = min(last, n - 1)
fn = self.reduce_check_table.get(lhs, None) fn = self.reduce_check_table.get(lhs, None)
if fn: if fn:
if fn(self, lhs, n, rule, ast, tokens, first, last): if fn(self, lhs, n, rule, ast, tokens, first, last):
@@ -1650,13 +1669,18 @@ class Python3Parser(PythonParser):
condition_jump2 = tokens[min(last - 1, len(tokens) - 1)] condition_jump2 = tokens[min(last - 1, len(tokens) - 1)]
# If there are two *distinct* condition jumps, they should not jump to the # If there are two *distinct* condition jumps, they should not jump to the
# same place. Otherwise we have some sort of "and"/"or". # same place. Otherwise we have some sort of "and"/"or".
if condition_jump2.kind.startswith("POP_JUMP_IF") and condition_jump != condition_jump2: if (
condition_jump2.kind.startswith("POP_JUMP_IF")
and condition_jump != condition_jump2
):
return condition_jump.attr == condition_jump2.attr return condition_jump.attr == condition_jump2.attr
if tokens[last] == "COME_FROM" and tokens[last].off2int() != condition_jump.attr: if (
tokens[last] == "COME_FROM"
and tokens[last].off2int() != condition_jump.attr
):
return False return False
# if condition_jump.attr < condition_jump2.off2int(): # if condition_jump.attr < condition_jump2.off2int():
# print("XXX", first, last) # print("XXX", first, last)
# for t in range(first, last): print(tokens[t]) # for t in range(first, last): print(tokens[t])
@@ -1678,7 +1702,6 @@ class Python3Parser(PythonParser):
< tokens[last].off2int() < tokens[last].off2int()
) )
elif lhs == "while1stmt": elif lhs == "while1stmt":
if while1stmt(self, lhs, n, rule, ast, tokens, first, last): if while1stmt(self, lhs, n, rule, ast, tokens, first, last):
return True return True
@@ -1700,7 +1723,6 @@ class Python3Parser(PythonParser):
return True return True
return False return False
elif lhs == "while1elsestmt": elif lhs == "while1elsestmt":
n = len(tokens) n = len(tokens)
if last == n: if last == n:
# Adjust for fuzziness in parsing # Adjust for fuzziness in parsing

View File

@@ -1,15 +1,13 @@
# Copyright (c) 2016 Rocky Bernstein # Copyright (c) 2016, 2024 Rocky Bernstein
""" """
spark grammar differences over Python 3.2 for Python 3.3. spark grammar differences over Python 3.2 for Python 3.3.
""" """
from __future__ import print_function
from uncompyle6.parser import PythonParserSingle from uncompyle6.parser import PythonParserSingle
from uncompyle6.parsers.parse32 import Python32Parser from uncompyle6.parsers.parse32 import Python32Parser
class Python33Parser(Python32Parser): class Python33Parser(Python32Parser):
def p_33on(self, args): def p_33on(self, args):
""" """
# Python 3.3+ adds yield from. # Python 3.3+ adds yield from.
@@ -19,13 +17,22 @@ class Python33Parser(Python32Parser):
""" """
def customize_grammar_rules(self, tokens, customize): def customize_grammar_rules(self, tokens, customize):
self.remove_rules(""" self.remove_rules(
"""
# 3.3+ adds POP_BLOCKS # 3.3+ adds POP_BLOCKS
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP
""") """
)
super(Python33Parser, self).customize_grammar_rules(tokens, customize) super(Python33Parser, self).customize_grammar_rules(tokens, customize)
# FIXME: move 3.3 stuff out of parse3.py and put it here.
# for i, token in enumerate(tokens):
# opname = token.kind
# opname_base = opname[: opname.rfind("_")]
return return
class Python33ParserSingle(Python33Parser, PythonParserSingle): class Python33ParserSingle(Python33Parser, PythonParserSingle):
pass pass

View File

@@ -35,22 +35,19 @@ Finally we save token information.
from __future__ import print_function from __future__ import print_function
import sys
from typing import Optional, Tuple from typing import Optional, Tuple
from xdis import iscode, instruction_size, Instruction
from xdis.bytecode import _get_const_info
from uncompyle6.scanners.tok import Token
from uncompyle6.scanner import parse_fn_counts_30_35
from uncompyle6.util import get_code_name
import xdis import xdis
# Get all the opcodes into globals # Get all the opcodes into globals
import xdis.opcodes.opcode_33 as op3 import xdis.opcodes.opcode_33 as op3
from xdis import Instruction, instruction_size, iscode
from xdis.bytecode import _get_const_info
from uncompyle6.scanner import Scanner, CONST_COLLECTIONS from uncompyle6.scanner import CONST_COLLECTIONS, Scanner, parse_fn_counts_30_35
from uncompyle6.scanners.tok import Token
import sys from uncompyle6.util import get_code_name
intern = sys.intern intern = sys.intern
@@ -485,6 +482,7 @@ class Scanner3(Scanner):
last_op_was_break = False last_op_was_break = False
new_tokens = [] new_tokens = []
operand_value = 0
for i, inst in enumerate(self.insts): for i, inst in enumerate(self.insts):
opname = inst.opname opname = inst.opname
@@ -536,10 +534,11 @@ class Scanner3(Scanner):
op = inst.opcode op = inst.opcode
if opname == "EXTENDED_ARG": if opname == "EXTENDED_ARG":
# FIXME: The EXTENDED_ARG is used to signal annotation if i + 1 < n:
# parameters operand_value = argval << 16
if i + 1 < n and self.insts[i + 1].opcode != self.opc.MAKE_FUNCTION:
continue continue
else:
operand_value = 0
if inst.offset in jump_targets: if inst.offset in jump_targets:
jump_idx = 0 jump_idx = 0
@@ -646,7 +645,7 @@ class Scanner3(Scanner):
attr = attr[:4] # remove last value: attr[5] == False attr = attr[:4] # remove last value: attr[5] == False
else: else:
pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35( pos_args, name_pair_args, annotate_args = parse_fn_counts_30_35(
inst.argval inst.argval + operand_value
) )
pattr = f"{pos_args} positional, {name_pair_args} keyword only, {annotate_args} annotated" pattr = f"{pos_args} positional, {name_pair_args} keyword only, {annotate_args} annotated"

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein # Copyright (c) 2015-2019, 2021-2022, 2024 by Rocky Bernstein
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@@ -19,20 +19,22 @@ This sets up opcodes Python's 3.3 and calls a generalized
scanner routine for Python 3. scanner routine for Python 3.
""" """
from __future__ import print_function
# bytecode verification, verify(), uses JUMP_OPs from here # bytecode verification, verify(), uses JUMP_OPs from here
from xdis.opcodes import opcode_33 as opc from xdis.opcodes import opcode_33 as opc
JUMP_OPS = opc.JUMP_OPS
from uncompyle6.scanners.scanner3 import Scanner3 from uncompyle6.scanners.scanner3 import Scanner3
class Scanner33(Scanner3):
JUMP_OPS = opc.JUMP_OPS
class Scanner33(Scanner3):
def __init__(self, show_asm=False, is_pypy=False): def __init__(self, show_asm=False, is_pypy=False):
Scanner3.__init__(self, (3, 3), show_asm) Scanner3.__init__(self, (3, 3), show_asm)
return return
pass pass
if __name__ == "__main__": if __name__ == "__main__":
from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str from xdis.version_info import PYTHON_VERSION_TRIPLE, version_tuple_to_str