jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 00:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

withasstmt -> with_as

Browse Source 
This matches Python's AST naamae better. Some linting and
sorting of dictionary keys done as well.
This commit is contained in:
rocky
2024-03-08 04:09:47 -05:00
parent 3724e02183
commit 156188f8bb
13 changed files with 356 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
uncompyle6/parsers/parse24.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2018, 2020, 2022-2023 Rocky Bernstein
# Copyright (c) 2016-2018, 2020, 2022-2024 Rocky Bernstein
"""
spark grammar differences over Python2.5 for Python 2.4.
"""
@@ -89,12 +89,14 @@ class Python24Parser(Python25Parser):
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
while1stmt ::= SETUP_LOOP returns COME_FROM
whilestmt ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM
with ::= expr setupwith SETUP_FINALLY suite_stmts_opt POP_BLOCK
LOAD_CONST COME_FROM with_cleanup
with_as ::= expr setupwithas store suite_stmts_opt POP_BLOCK
LOAD_CONST COME_FROM with_cleanup
with_cleanup ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
with_cleanup ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY
withasstmt ::= expr setupwithas store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM with_cleanup
with ::= expr setupwith SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM with_cleanup
stmt ::= with
stmt ::= withasstmt
stmt ::= with
stmt ::= with_as
"""
)
super(Python24Parser, self).customize_grammar_rules(tokens, customize)

8
uncompyle6/parsers/parse25.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2018, 2020, 2022 Rocky Bernstein
# Copyright (c) 2016-2018, 2020, 2022, 2024 Rocky Bernstein
"""
spark grammar differences over Python2.6 for Python 2.5.
"""
@@ -33,7 +33,7 @@ class Python25Parser(Python26Parser):
POP_BLOCK LOAD_CONST COME_FROM with_cleanup
# Semantic actions want store to be at index 2
withasstmt ::= expr setupwithas store suite_stmts_opt
with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM with_cleanup
@@ -48,7 +48,7 @@ class Python25Parser(Python26Parser):
# Python 2.6 omits the LOAD_FAST DELETE_FAST below
# withas is allowed as a "from future" in 2.5
withasstmt ::= expr setupwithas store suite_stmts_opt
with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM
with_cleanup
@@ -67,7 +67,7 @@ class Python25Parser(Python26Parser):
setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
with ::= expr setupwith SETUP_FINALLY suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
withasstmt ::= expr setupwithas store suite_stmts_opt
with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
classdefdeco ::= classdefdeco1 store

12
uncompyle6/parsers/parse26.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2023 Rocky Bernstein
# Copyright (c) 2017-2024 Rocky Bernstein
"""
spark grammar differences over Python2 for Python 2.6.
"""
@@ -136,7 +136,7 @@ class Python26Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# Semantic actions want store to be at index 2
withasstmt ::= expr setupwithas store suite_stmts_opt
with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# This is truly weird. 2.7 does this (not including POP_TOP) with
@@ -352,9 +352,9 @@ class Python26Parser(Python2Parser):
def customize_grammar_rules(self, tokens, customize):
self.remove_rules(
"""
withasstmt ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
"""
)
super(Python26Parser, self).customize_grammar_rules(tokens, customize)
@@ -391,7 +391,6 @@ class Python26Parser(Python2Parser):
("and", ("expr", "jmp_false", "expr", "come_from_opt")),
("assert_expr_and", ("assert_expr", "jmp_false", "expr")),
):
# FIXME: workaround profiling bug
if ast[1] is None:
return False
@@ -491,7 +490,6 @@ class Python26Parser(Python2Parser):
("JUMP_FORWARD", "RETURN_VALUE")
) or (tokens[last - 3] == "JUMP_FORWARD" and tokens[last - 3].attr != 2)
elif lhs == "tryelsestmt":
# We need to distinguish "try_except" from "tryelsestmt"; we do that
# by making sure that the jump before the except handler jumps to
# code somewhere before the end of the construct.

6
uncompyle6/parsers/parse27.py
View File

@@ -161,9 +161,9 @@ class Python27Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
whilestmt ::= SETUP_LOOP testexpr returns
_come_froms POP_BLOCK COME_FROM

6
uncompyle6/parsers/parse3.py
View File

@@ -287,9 +287,9 @@ class Python3Parser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
expr_jt ::= expr jmp_true
expr_jitop ::= expr JUMP_IF_TRUE_OR_POP

17
uncompyle6/parsers/parse30.py
View File

@@ -66,7 +66,7 @@ class Python30Parser(Python31Parser):
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
withasstmt ::= expr setupwithas store suite_stmts_opt
with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_FINALLY
LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
setupwithas ::= DUP_TOP LOAD_ATTR STORE_FAST LOAD_ATTR CALL_FUNCTION_0 setup_finally
@@ -222,12 +222,17 @@ class Python30Parser(Python31Parser):
# The were found using grammar coverage
while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK COME_FROM_LOOP
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
else_suitel COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK
COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP
withasstmt ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
with_as ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST
COME_FROM_WITH WITH_CLEANUP END_FINALLY
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
COME_FROM_WITH WITH_CLEANUP END_FINALLY
# lc_body ::= LOAD_FAST expr LIST_APPEND
# lc_body ::= LOAD_NAME expr LIST_APPEND

2
uncompyle6/parsers/parse31.py
View File

@@ -23,7 +23,7 @@ class Python31Parser(Python32Parser):
# Keeps Python 3.1 "with .. as" designator in the same position as it is in other version.
setupwithas31 ::= setupwithas SETUP_FINALLY load delete
withasstmt ::= expr setupwithas31 store
with_as ::= expr setupwithas31 store
suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_FINALLY
load delete WITH_CLEANUP END_FINALLY

122
uncompyle6/parsers/parse35.py
View File

@@ -1,15 +1,17 @@
# Copyright (c) 2016-2017, 2019, 2021, 2023 Rocky Bernstein
# Copyright (c) 2016-2017, 2019, 2021, 2023-2024
# Rocky Bernstein
"""
spark grammar differences over Python 3.4 for Python 3.5.
"""
from __future__ import print_function
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse34 import Python34Parser
class Python35Parser(Python34Parser):
class Python35Parser(Python34Parser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python35Parser, self).__init__(debug_parser)
self.customized = {}
@@ -55,7 +57,7 @@ class Python35Parser(Python34Parser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
withasstmt ::= expr
with_as ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
@@ -135,40 +137,42 @@ class Python35Parser(Python34Parser):
"""
def customize_grammar_rules(self, tokens, customize):
self.remove_rules("""
self.remove_rules(
"""
yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
yield_from ::= expr expr YIELD_FROM
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt
with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
""")
"""
)
super(Python35Parser, self).customize_grammar_rules(tokens, customize)
for i, token in enumerate(tokens):
opname = token.kind
if opname == 'LOAD_ASSERT':
if 'PyPy' in customize:
if opname == "LOAD_ASSERT":
if "PyPy" in customize:
rules_str = """
stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
"""
self.add_unique_doc_rules(rules_str, customize)
# FIXME: I suspect this is wrong for 3.6 and 3.5, but
# I haven't verified what the 3.7ish fix is
elif opname == 'BUILD_MAP_UNPACK_WITH_CALL':
elif opname == "BUILD_MAP_UNPACK_WITH_CALL":
if self.version < (3, 7):
self.addRule("expr ::= unmapexpr", nop_func)
nargs = token.attr % 256
map_unpack_n = "map_unpack_%s" % nargs
rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs)
rule = map_unpack_n + " ::= " + "expr " * (nargs)
self.addRule(rule, nop_func)
rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname)
self.addRule(rule, nop_func)
call_token = tokens[i+1]
rule = 'call ::= expr unmapexpr ' + call_token.kind
call_token = tokens[i + 1]
rule = "call ::= expr unmapexpr " + call_token.kind
self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH' and self.version < (3, 8):
elif opname == "BEFORE_ASYNC_WITH" and self.version < (3, 8):
# Some Python 3.5+ async additions
rules_str = """
stmt ::= async_with_stmt
@@ -199,24 +203,27 @@ class Python35Parser(Python34Parser):
async_with_post
"""
self.addRule(rules_str, nop_func)
elif opname == 'BUILD_MAP_UNPACK':
self.addRule("""
elif opname == "BUILD_MAP_UNPACK":
self.addRule(
"""
expr ::= dict_unpack
dict_unpack ::= dict_comp BUILD_MAP_UNPACK
""", nop_func)
""",
nop_func,
)
elif opname == 'SETUP_WITH':
elif opname == "SETUP_WITH":
# Python 3.5+ has WITH_CLEANUP_START/FINISH
rules_str = """
with ::= expr
SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
with ::= expr
SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
withasstmt ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
with_as ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
pass
@@ -230,19 +237,24 @@ class Python35Parser(Python34Parser):
# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
# 2 for * and ** args (CALL_FUNCTION_VAR_KW).
# Yes, this computation based on instruction name is a little bit hoaky.
nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
nak = (len(opname) - len("CALL_FUNCTION")) // 3
uniq_param = args_kw + args_pos
if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops):
rule = ('async_call ::= expr ' +
('pos_arg ' * args_pos) +
('kwarg ' * args_kw) +
'expr ' * nak + token.kind +
' GET_AWAITABLE LOAD_CONST YIELD_FROM')
if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
rule = (
"async_call ::= expr "
+ ("pos_arg " * args_pos)
+ ("kwarg " * args_kw)
+ "expr " * nak
+ token.kind
+ " GET_AWAITABLE LOAD_CONST YIELD_FROM"
)
self.add_unique_rule(rule, token.kind, uniq_param, customize)
self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
self.add_unique_rule(
"expr ::= async_call", token.kind, uniq_param, customize
)
if opname.startswith('CALL_FUNCTION_VAR'):
if opname.startswith("CALL_FUNCTION_VAR"):
# Python 3.5 changes the stack position of *args. KW args come
# after *args.
@@ -250,43 +262,55 @@ class Python35Parser(Python34Parser):
# CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
token.kind = self.call_fn_name(token)
if opname.endswith('KW'):
kw = 'expr '
if opname.endswith("KW"):
kw = "expr "
else:
kw = ''
rule = ('call ::= expr expr ' +
('pos_arg ' * args_pos) +
('kwarg ' * args_kw) + kw + token.kind)
kw = ""
rule = (
"call ::= expr expr "
+ ("pos_arg " * args_pos)
+ ("kwarg " * args_kw)
+ kw
+ token.kind
)
# Note: semantic actions make use of the fact of whether "args_pos"
# zero or not in creating a template rule.
self.add_unique_rule(rule, token.kind, args_pos, customize)
else:
super(Python35Parser, self).custom_classfunc_rule(opname, token, customize, *args
super(Python35Parser, self).custom_classfunc_rule(
opname, token, customize, *args
)
class Python35ParserSingle(Python35Parser, PythonParserSingle):
pass
if __name__ == '__main__':
if __name__ == "__main__":
# Check grammar
p = Python35Parser()
p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 5):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
opcode_set = set(s.opc.opname).union(
set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
""".split()
)
)
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))

217
uncompyle6/parsers/parse36.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2020, 2022-2023 Rocky Bernstein
# Copyright (c) 2016-2020, 2022-2024 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -17,24 +17,25 @@ spark grammar differences over Python 3.5 for Python 3.6.
"""
from __future__ import print_function
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse35 import Python35Parser
from uncompyle6.scanners.tok import Token
class Python36Parser(Python35Parser):
class Python36Parser(Python35Parser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python36Parser, self).__init__(debug_parser)
self.customized = {}
def p_36_jump(self, args):
"""
# Zero or one COME_FROM
# And/or expressions have this
come_from_opt ::= COME_FROM?
"""
def p_36_misc(self, args):
"""sstmt ::= sstmt RETURN_LAST
@@ -207,7 +208,8 @@ class Python36Parser(Python35Parser):
# self.remove_rules("""
# """)
super(Python36Parser, self).customize_grammar_rules(tokens, customize)
self.remove_rules("""
self.remove_rules(
"""
_ifstmts_jumpl ::= c_stmts_opt
_ifstmts_jumpl ::= _ifstmts_jump
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM
@@ -234,7 +236,8 @@ class Python36Parser(Python35Parser):
for_block pb_ja
else_suite COME_FROM_LOOP
""")
"""
)
self.check_reduce["call_kw"] = "AST"
# Opcode names in the custom_ops_processed set have rules that get added
@@ -247,24 +250,23 @@ class Python36Parser(Python35Parser):
# the start.
custom_ops_processed = set()
for i, token in enumerate(tokens):
opname = token.kind
if opname == 'FORMAT_VALUE':
if opname == "FORMAT_VALUE":
rules_str = """
expr ::= formatted_value1
formatted_value1 ::= expr FORMAT_VALUE
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == 'FORMAT_VALUE_ATTR':
elif opname == "FORMAT_VALUE_ATTR":
rules_str = """
expr ::= formatted_value2
formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname == 'MAKE_FUNCTION_CLOSURE':
if 'LOAD_DICTCOMP' in self.seen_ops:
elif opname == "MAKE_FUNCTION_CLOSURE":
if "LOAD_DICTCOMP" in self.seen_ops:
# Is there something general going on here?
rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
@@ -272,7 +274,7 @@ class Python36Parser(Python35Parser):
GET_ITER CALL_FUNCTION_1
"""
self.addRule(rule, nop_func)
elif 'LOAD_SETCOMP' in self.seen_ops:
elif "LOAD_SETCOMP" in self.seen_ops:
rule = """
set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
MAKE_FUNCTION_CLOSURE expr
@@ -280,7 +282,7 @@ class Python36Parser(Python35Parser):
"""
self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH':
elif opname == "BEFORE_ASYNC_WITH":
rules_str = """
stmt ::= async_with_stmt
async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
@@ -306,30 +308,37 @@ class Python36Parser(Python35Parser):
"""
self.addRule(rules_str, nop_func)
elif opname.startswith('BUILD_STRING'):
elif opname.startswith("BUILD_STRING"):
v = token.attr
rules_str = """
expr ::= joined_str
joined_str ::= %sBUILD_STRING_%d
""" % ("expr " * v, v)
""" % (
"expr " * v,
v,
)
self.add_unique_doc_rules(rules_str, customize)
if 'FORMAT_VALUE_ATTR' in self.seen_ops:
if "FORMAT_VALUE_ATTR" in self.seen_ops:
rules_str = """
formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
expr ::= formatted_value_attr
"""
self.add_unique_doc_rules(rules_str, customize)
elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
v = token.attr
rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)
rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
self.addRule(rule, nop_func)
elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
v = token.attr
rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
'expr32 ' * int((v//32) % 32) +
'expr ' * (v % 32) + opname)
rule = (
"build_tuple_unpack_with_call ::= "
+ "expr1024 " * int(v // 1024)
+ "expr32 " * int((v // 32) % 32)
+ "expr " * (v % 32)
+ opname
)
self.addRule(rule, nop_func)
rule = ('starred ::= %s %s' % ('expr ' * v, opname))
rule = "starred ::= %s %s" % ("expr " * v, opname)
self.addRule(rule, nop_func)
elif opname == "GET_AITER":
self.addRule(
@@ -475,7 +484,6 @@ class Python36Parser(Python35Parser):
)
custom_ops_processed.add(opname)
elif opname == "GET_ANEXT":
self.addRule(
"""
@@ -500,7 +508,7 @@ class Python36Parser(Python35Parser):
)
custom_ops_processed.add(opname)
elif opname == 'SETUP_ANNOTATIONS':
elif opname == "SETUP_ANNOTATIONS":
# 3.6 Variable Annotations PEP 526
# This seems to come before STORE_ANNOTATION, and doesn't
# correspond to direct Python source code.
@@ -516,7 +524,7 @@ class Python36Parser(Python35Parser):
"""
self.addRule(rule, nop_func)
# Check to combine assignment + annotation into one statement
self.check_reduce['assign'] = 'token'
self.check_reduce["assign"] = "token"
elif opname == "WITH_CLEANUP_START":
rules_str = """
stmt ::= with_null
@@ -524,13 +532,13 @@ class Python36Parser(Python35Parser):
with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
elif opname == 'SETUP_WITH':
elif opname == "SETUP_WITH":
rules_str = """
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
with_suffix
# Removes POP_BLOCK LOAD_CONST from 3.6-
withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_suffix
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_WITH
@@ -542,7 +550,6 @@ class Python36Parser(Python35Parser):
return
def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
args_pos, args_kw = self.get_pos_kw(token)
# Additional exprs for * and ** args:
@@ -550,140 +557,186 @@ class Python36Parser(Python35Parser):
# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
# 2 for * and ** args (CALL_FUNCTION_VAR_KW).
# Yes, this computation based on instruction name is a little bit hoaky.
nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
nak = (len(opname) - len("CALL_FUNCTION")) // 3
uniq_param = args_kw + args_pos
if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops):
rule = ('async_call ::= expr ' +
('pos_arg ' * args_pos) +
('kwarg ' * args_kw) +
'expr ' * nak + token.kind +
' GET_AWAITABLE LOAD_CONST YIELD_FROM')
if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
rule = (
"async_call ::= expr "
+ ("pos_arg " * args_pos)
+ ("kwarg " * args_kw)
+ "expr " * nak
+ token.kind
+ " GET_AWAITABLE LOAD_CONST YIELD_FROM"
)
self.add_unique_rule(rule, token.kind, uniq_param, customize)
self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
self.add_unique_rule(
"expr ::= async_call", token.kind, uniq_param, customize
)
if opname.startswith('CALL_FUNCTION_KW'):
if opname.startswith("CALL_FUNCTION_KW"):
if is_pypy:
# PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions
super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
super(Python36Parser, self).custom_classfunc_rule(
opname, token, customize, next_token, is_pypy
)
else:
self.addRule("expr ::= call_kw36", nop_func)
values = 'expr ' * token.attr
rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
values = "expr " * token.attr
rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(
**locals()
)
self.add_unique_rule(rule, token.kind, token.attr, customize)
elif opname == 'CALL_FUNCTION_EX_KW':
elif opname == "CALL_FUNCTION_EX_KW":
# Note: this doesn't exist in 3.7 and later
self.addRule("""expr ::= call_ex_kw4
self.addRule(
"""expr ::= call_ex_kw4
call_ex_kw4 ::= expr
expr
expr
CALL_FUNCTION_EX_KW
""",
nop_func)
if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_op_basenames:
self.addRule("""expr ::= call_ex_kw
nop_func,
)
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
self.addRule(
"""expr ::= call_ex_kw
call_ex_kw ::= expr expr build_map_unpack_with_call
CALL_FUNCTION_EX_KW
""", nop_func)
if 'BUILD_TUPLE_UNPACK_WITH_CALL' in self.seen_op_basenames:
""",
nop_func,
)
if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
# FIXME: should this be parameterized by EX value?
self.addRule("""expr ::= call_ex_kw3
self.addRule(
"""expr ::= call_ex_kw3
call_ex_kw3 ::= expr
build_tuple_unpack_with_call
expr
CALL_FUNCTION_EX_KW
""", nop_func)
if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_op_basenames:
""",
nop_func,
)
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
# FIXME: should this be parameterized by EX value?
self.addRule("""expr ::= call_ex_kw2
self.addRule(
"""expr ::= call_ex_kw2
call_ex_kw2 ::= expr
build_tuple_unpack_with_call
build_map_unpack_with_call
CALL_FUNCTION_EX_KW
""", nop_func)
""",
nop_func,
)
elif opname == 'CALL_FUNCTION_EX':
self.addRule("""
elif opname == "CALL_FUNCTION_EX":
self.addRule(
"""
expr ::= call_ex
starred ::= expr
call_ex ::= expr starred CALL_FUNCTION_EX
""", nop_func)
""",
nop_func,
)
if self.version >= (3, 6):
if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_ops:
self.addRule("""
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
self.addRule(
"""
expr ::= call_ex_kw
call_ex_kw ::= expr expr
build_map_unpack_with_call CALL_FUNCTION_EX
""", nop_func)
if 'BUILD_TUPLE_UNPACK_WITH_CALL' in self.seen_ops:
self.addRule("""
""",
nop_func,
)
if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
self.addRule(
"""
expr ::= call_ex_kw3
call_ex_kw3 ::= expr
build_tuple_unpack_with_call
%s
CALL_FUNCTION_EX
""" % 'expr ' * token.attr, nop_func)
"""
% "expr "
* token.attr,
nop_func,
)
pass
# FIXME: Is this right?
self.addRule("""
self.addRule(
"""
expr ::= call_ex_kw4
call_ex_kw4 ::= expr
expr
expr
CALL_FUNCTION_EX
""", nop_func)
""",
nop_func,
)
pass
else:
super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
super(Python36Parser, self).custom_classfunc_rule(
opname, token, customize, next_token, is_pypy
)
def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python36Parser,
self).reduce_is_invalid(rule, ast,
tokens, first, last)
invalid = super(Python36Parser, self).reduce_is_invalid(
rule, ast, tokens, first, last
)
if invalid:
return invalid
if rule[0] == 'assign':
if rule[0] == "assign":
# Try to combine assignment + annotation into one statement
if (len(tokens) >= last + 1 and
tokens[last] == 'LOAD_NAME' and
tokens[last+1] == 'STORE_ANNOTATION' and
tokens[last-1].pattr == tokens[last+1].pattr):
if (
len(tokens) >= last + 1
and tokens[last] == "LOAD_NAME"
and tokens[last + 1] == "STORE_ANNOTATION"
and tokens[last - 1].pattr == tokens[last + 1].pattr
):
# Will handle as ann_assign_init_value
return True
pass
if rule[0] == 'call_kw':
if rule[0] == "call_kw":
# Make sure we don't derive call_kw
nt = ast[0]
while not isinstance(nt, Token):
if nt[0] == 'call_kw':
if nt[0] == "call_kw":
return True
nt = nt[0]
pass
pass
return False
class Python36ParserSingle(Python36Parser, PythonParserSingle):
pass
if __name__ == '__main__':
if __name__ == "__main__":
# Check grammar
p = Python36Parser()
p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 6):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
opcode_set = set(s.opc.opname).union(
set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
""".split()
)
)
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))

19
uncompyle6/parsers/parse37.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2020, 2022-2023 Rocky Bernstein
# Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -17,10 +17,12 @@ Python 3.7 grammar for the spark Earley-algorithm parser.
"""
from __future__ import print_function
from uncompyle6.scanners.tok import Token
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse37base import Python37BaseParser
from uncompyle6.scanners.tok import Token
class Python37Parser(Python37BaseParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -249,8 +251,7 @@ class Python37Parser(Python37BaseParser):
"""
def p_generator_exp(self, args):
"""
"""
""" """
def p_jump(self, args):
"""
@@ -757,7 +758,7 @@ class Python37Parser(Python37BaseParser):
"""
def p_dict_comp3(self, args):
""""
""" "
expr ::= dict_comp
stmt ::= dict_comp_func
@@ -1554,7 +1555,7 @@ class Python37Parser(Python37BaseParser):
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
# Removes POP_BLOCK LOAD_CONST from 3.6-
withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
if self.version < (3, 8):
@@ -1575,7 +1576,6 @@ class Python37Parser(Python37BaseParser):
pass
def custom_classfunc_rule(self, opname, token, customize, next_token):
args_pos, args_kw = self.get_pos_kw(token)
# Additional exprs for * and ** args:
@@ -1718,6 +1718,7 @@ class Python37Parser(Python37BaseParser):
pass
return False
def info(args):
# Check grammar
p = Python37Parser()
@@ -1748,7 +1749,7 @@ if __name__ == "__main__":
# FIXME: DRY this with other parseXX.py routines
p = Python37Parser()
p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 7):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()

25
uncompyle6/parsers/parse37base.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2017, 2019-2020, 2022-2023 Rocky Bernstein
# Copyright (c) 2016-2017, 2019-2020, 2022-2024 Rocky Bernstein
"""
Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
"""
@@ -1055,14 +1055,14 @@ class Python37BaseParser(PythonParser):
elif opname == "SETUP_WITH":
rules_str = """
stmt ::= with
stmt ::= withasstmt
stmt ::= with_as
with ::= expr
SETUP_WITH POP_TOP
suite_stmts_opt
COME_FROM_WITH
with_suffix
withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_suffix
with ::= expr
@@ -1071,7 +1071,7 @@ class Python37BaseParser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix
withasstmt ::= expr
with_as ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix
@@ -1080,7 +1080,7 @@ class Python37BaseParser(PythonParser):
SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix
withasstmt ::= expr
with_as ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix
@@ -1098,17 +1098,18 @@ class Python37BaseParser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix
withasstmt ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
withasstmt ::= expr
SETUP_WITH store suite_stmts
POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_WITH
with_suffix
with_as ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_as ::= expr
SETUP_WITH store suite_stmts
POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
"""
self.addRule(rules_str, nop_func)