jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-03 00:45:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #489 from rocky/withasstmt-to-with_as

Browse Source 
withasstmt -> with_as
This commit is contained in:
R. Bernstein
2024-03-08 04:12:55 -05:00
committed by GitHub
parent 3724e02183 156188f8bb
commit e6ff6033cf
13 changed files with 356 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
uncompyle6/parser.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2023 Rocky Bernstein # Copyright (c) 2015-2024 Rocky Bernstein
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org> # Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com> # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# Copyright (c) 1999 John Aycock # Copyright (c) 1999 John Aycock
@@ -21,10 +21,11 @@ Common uncompyle6 parser routines.
import sys import sys
from spark_parser import GenericASTBuilder, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG, GenericASTBuilder
from uncompyle6.show import maybe_show_asm
from xdis import iscode from xdis import iscode
from uncompyle6.show import maybe_show_asm
class ParserError(Exception): class ParserError(Exception):
def __init__(self, token, offset, debug=PARSER_DEFAULT_DEBUG): def __init__(self, token, offset, debug=PARSER_DEFAULT_DEBUG):
@@ -91,7 +92,14 @@ class PythonParser(GenericASTBuilder):
# singleton reduction that we can simplify. It also happens to be optional # singleton reduction that we can simplify. It also happens to be optional
# in its other derivation # in its other derivation
self.optional_nt |= frozenset( self.optional_nt |= frozenset(
("come_froms", "suite_stmts", "l_stmts_opt", "c_stmts_opt", "stmts_opt", "stmt") (
"come_froms",
"suite_stmts",
"l_stmts_opt",
"c_stmts_opt",
"stmts_opt",
"stmt",
)
) )
# Reduce singleton reductions in these nonterminals: # Reduce singleton reductions in these nonterminals:
@@ -113,10 +121,10 @@ class PythonParser(GenericASTBuilder):
def add_unique_rule(self, rule, opname, arg_count, customize): def add_unique_rule(self, rule, opname, arg_count, customize):
"""Add rule to grammar, but only if it hasn't been added previously """Add rule to grammar, but only if it hasn't been added previously
opname and stack_count are used in the customize() semantic opname and stack_count are used in the customize() semantic
the actions to add the semantic action rule. Stack_count is the actions to add the semantic action rule. Stack_count is
used in custom opcodes like MAKE_FUNCTION to indicate how used in custom opcodes like MAKE_FUNCTION to indicate how
many arguments it has. Often it is not used. many arguments it has. Often it is not used.
""" """
if rule not in self.new_rules: if rule not in self.new_rules:
# print("XXX ", rule) # debug # print("XXX ", rule) # debug
@@ -223,7 +231,9 @@ class PythonParser(GenericASTBuilder):
""" """
# Low byte indicates number of positional parameters, # Low byte indicates number of positional parameters,
# high byte number of keyword parameters # high byte number of keyword parameters
assert token.kind.startswith("CALL_FUNCTION") or token.kind.startswith("CALL_METHOD") assert token.kind.startswith("CALL_FUNCTION") or token.kind.startswith(
"CALL_METHOD"
)
args_pos = token.attr & 0xFF args_pos = token.attr & 0xFF
args_kw = (token.attr >> 8) & 0xFF args_kw = (token.attr >> 8) & 0xFF
return args_pos, args_kw return args_pos, args_kw
@@ -364,7 +374,7 @@ class PythonParser(GenericASTBuilder):
stmt ::= tryelsestmt stmt ::= tryelsestmt
stmt ::= tryfinallystmt stmt ::= tryfinallystmt
stmt ::= with stmt ::= with
stmt ::= withasstmt stmt ::= with_as
stmt ::= delete stmt ::= delete
delete ::= DELETE_FAST delete ::= DELETE_FAST
@@ -907,7 +917,7 @@ def python_parser(
if __name__ == "__main__": if __name__ == "__main__":
def parse_test(co): def parse_test(co):
from xdis import PYTHON_VERSION_TRIPLE, IS_PYPY from xdis import IS_PYPY, PYTHON_VERSION_TRIPLE
ast = python_parser(PYTHON_VERSION_TRIPLE, co, showasm=True, is_pypy=IS_PYPY) ast = python_parser(PYTHON_VERSION_TRIPLE, co, showasm=True, is_pypy=IS_PYPY)
print(ast) print(ast)

12
uncompyle6/parsers/parse24.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2018, 2020, 2022-2023 Rocky Bernstein # Copyright (c) 2016-2018, 2020, 2022-2024 Rocky Bernstein
""" """
spark grammar differences over Python2.5 for Python 2.4. spark grammar differences over Python2.5 for Python 2.4.
""" """
@@ -89,12 +89,14 @@ class Python24Parser(Python25Parser):
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
while1stmt ::= SETUP_LOOP returns COME_FROM while1stmt ::= SETUP_LOOP returns COME_FROM
whilestmt ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM whilestmt ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM
with ::= expr setupwith SETUP_FINALLY suite_stmts_opt POP_BLOCK
LOAD_CONST COME_FROM with_cleanup
with_as ::= expr setupwithas store suite_stmts_opt POP_BLOCK
LOAD_CONST COME_FROM with_cleanup
with_cleanup ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY with_cleanup ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
with_cleanup ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY with_cleanup ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY
withasstmt ::= expr setupwithas store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM with_cleanup stmt ::= with
with ::= expr setupwith SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM with_cleanup stmt ::= with_as
stmt ::= with
stmt ::= withasstmt
""" """
) )
super(Python24Parser, self).customize_grammar_rules(tokens, customize) super(Python24Parser, self).customize_grammar_rules(tokens, customize)

8
uncompyle6/parsers/parse25.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2018, 2020, 2022 Rocky Bernstein # Copyright (c) 2016-2018, 2020, 2022, 2024 Rocky Bernstein
""" """
spark grammar differences over Python2.6 for Python 2.5. spark grammar differences over Python2.6 for Python 2.5.
""" """
@@ -33,7 +33,7 @@ class Python25Parser(Python26Parser):
POP_BLOCK LOAD_CONST COME_FROM with_cleanup POP_BLOCK LOAD_CONST COME_FROM with_cleanup
# Semantic actions want store to be at index 2 # Semantic actions want store to be at index 2
withasstmt ::= expr setupwithas store suite_stmts_opt with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM with_cleanup POP_BLOCK LOAD_CONST COME_FROM with_cleanup
@@ -48,7 +48,7 @@ class Python25Parser(Python26Parser):
# Python 2.6 omits the LOAD_FAST DELETE_FAST below # Python 2.6 omits the LOAD_FAST DELETE_FAST below
# withas is allowed as a "from future" in 2.5 # withas is allowed as a "from future" in 2.5
withasstmt ::= expr setupwithas store suite_stmts_opt with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM POP_BLOCK LOAD_CONST COME_FROM
with_cleanup with_cleanup
@@ -67,7 +67,7 @@ class Python25Parser(Python26Parser):
setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP setupwith ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
with ::= expr setupwith SETUP_FINALLY suite_stmts_opt with ::= expr setupwith SETUP_FINALLY suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
withasstmt ::= expr setupwithas store suite_stmts_opt with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
classdefdeco ::= classdefdeco1 store classdefdeco ::= classdefdeco1 store

12
uncompyle6/parsers/parse26.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2023 Rocky Bernstein # Copyright (c) 2017-2024 Rocky Bernstein
""" """
spark grammar differences over Python2 for Python 2.6. spark grammar differences over Python2 for Python 2.6.
""" """
@@ -136,7 +136,7 @@ class Python26Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# Semantic actions want store to be at index 2 # Semantic actions want store to be at index 2
withasstmt ::= expr setupwithas store suite_stmts_opt with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
# This is truly weird. 2.7 does this (not including POP_TOP) with # This is truly weird. 2.7 does this (not including POP_TOP) with
@@ -352,9 +352,9 @@ class Python26Parser(Python2Parser):
def customize_grammar_rules(self, tokens, customize): def customize_grammar_rules(self, tokens, customize):
self.remove_rules( self.remove_rules(
""" """
withasstmt ::= expr SETUP_WITH store suite_stmts_opt with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
""" """
) )
super(Python26Parser, self).customize_grammar_rules(tokens, customize) super(Python26Parser, self).customize_grammar_rules(tokens, customize)
@@ -391,7 +391,6 @@ class Python26Parser(Python2Parser):
("and", ("expr", "jmp_false", "expr", "come_from_opt")), ("and", ("expr", "jmp_false", "expr", "come_from_opt")),
("assert_expr_and", ("assert_expr", "jmp_false", "expr")), ("assert_expr_and", ("assert_expr", "jmp_false", "expr")),
): ):
# FIXME: workaround profiling bug # FIXME: workaround profiling bug
if ast[1] is None: if ast[1] is None:
return False return False
@@ -491,7 +490,6 @@ class Python26Parser(Python2Parser):
("JUMP_FORWARD", "RETURN_VALUE") ("JUMP_FORWARD", "RETURN_VALUE")
) or (tokens[last - 3] == "JUMP_FORWARD" and tokens[last - 3].attr != 2) ) or (tokens[last - 3] == "JUMP_FORWARD" and tokens[last - 3].attr != 2)
elif lhs == "tryelsestmt": elif lhs == "tryelsestmt":
# We need to distinguish "try_except" from "tryelsestmt"; we do that # We need to distinguish "try_except" from "tryelsestmt"; we do that
# by making sure that the jump before the except handler jumps to # by making sure that the jump before the except handler jumps to
# code somewhere before the end of the construct. # code somewhere before the end of the construct.

6
uncompyle6/parsers/parse27.py
View File

@@ -161,9 +161,9 @@ class Python27Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
whilestmt ::= SETUP_LOOP testexpr returns whilestmt ::= SETUP_LOOP testexpr returns
_come_froms POP_BLOCK COME_FROM _come_froms POP_BLOCK COME_FROM

6
uncompyle6/parsers/parse3.py
View File

@@ -287,9 +287,9 @@ class Python3Parser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
expr_jt ::= expr jmp_true expr_jt ::= expr jmp_true
expr_jitop ::= expr JUMP_IF_TRUE_OR_POP expr_jitop ::= expr JUMP_IF_TRUE_OR_POP

17
uncompyle6/parsers/parse30.py
View File

@@ -66,7 +66,7 @@ class Python30Parser(Python31Parser):
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
withasstmt ::= expr setupwithas store suite_stmts_opt with_as ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_FINALLY POP_BLOCK LOAD_CONST COME_FROM_FINALLY
LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
setupwithas ::= DUP_TOP LOAD_ATTR STORE_FAST LOAD_ATTR CALL_FUNCTION_0 setup_finally setupwithas ::= DUP_TOP LOAD_ATTR STORE_FAST LOAD_ATTR CALL_FUNCTION_0 setup_finally
@@ -222,12 +222,17 @@ class Python30Parser(Python31Parser):
# The were found using grammar coverage # The were found using grammar coverage
while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP else_suitel COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK
COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP
withasstmt ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY with_as ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY COME_FROM_WITH WITH_CLEANUP END_FINALLY
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
COME_FROM_WITH WITH_CLEANUP END_FINALLY
# lc_body ::= LOAD_FAST expr LIST_APPEND # lc_body ::= LOAD_FAST expr LIST_APPEND
# lc_body ::= LOAD_NAME expr LIST_APPEND # lc_body ::= LOAD_NAME expr LIST_APPEND

2
uncompyle6/parsers/parse31.py
View File

@@ -23,7 +23,7 @@ class Python31Parser(Python32Parser):
# Keeps Python 3.1 "with .. as" designator in the same position as it is in other version. # Keeps Python 3.1 "with .. as" designator in the same position as it is in other version.
setupwithas31 ::= setupwithas SETUP_FINALLY load delete setupwithas31 ::= setupwithas SETUP_FINALLY load delete
withasstmt ::= expr setupwithas31 store with_as ::= expr setupwithas31 store
suite_stmts_opt suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_FINALLY POP_BLOCK LOAD_CONST COME_FROM_FINALLY
load delete WITH_CLEANUP END_FINALLY load delete WITH_CLEANUP END_FINALLY

122
uncompyle6/parsers/parse35.py
View File

@@ -1,15 +1,17 @@
# Copyright (c) 2016-2017, 2019, 2021, 2023 Rocky Bernstein # Copyright (c) 2016-2017, 2019, 2021, 2023-2024
# Rocky Bernstein
""" """
spark grammar differences over Python 3.4 for Python 3.5. spark grammar differences over Python 3.4 for Python 3.5.
""" """
from __future__ import print_function from __future__ import print_function
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse34 import Python34Parser from uncompyle6.parsers.parse34 import Python34Parser
class Python35Parser(Python34Parser):
class Python35Parser(Python34Parser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python35Parser, self).__init__(debug_parser) super(Python35Parser, self).__init__(debug_parser)
self.customized = {} self.customized = {}
@@ -55,7 +57,7 @@ class Python35Parser(Python34Parser):
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
withasstmt ::= expr with_as ::= expr
SETUP_WITH store suite_stmts_opt SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
@@ -135,40 +137,42 @@ class Python35Parser(Python34Parser):
""" """
def customize_grammar_rules(self, tokens, customize): def customize_grammar_rules(self, tokens, customize):
self.remove_rules(""" self.remove_rules(
"""
yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
yield_from ::= expr expr YIELD_FROM yield_from ::= expr expr YIELD_FROM
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt with ::= expr SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
withasstmt ::= expr SETUP_WITH store suite_stmts_opt with_as ::= expr SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY WITH_CLEANUP END_FINALLY
""") """
)
super(Python35Parser, self).customize_grammar_rules(tokens, customize) super(Python35Parser, self).customize_grammar_rules(tokens, customize)
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
opname = token.kind opname = token.kind
if opname == 'LOAD_ASSERT': if opname == "LOAD_ASSERT":
if 'PyPy' in customize: if "PyPy" in customize:
rules_str = """ rules_str = """
stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
""" """
self.add_unique_doc_rules(rules_str, customize) self.add_unique_doc_rules(rules_str, customize)
# FIXME: I suspect this is wrong for 3.6 and 3.5, but # FIXME: I suspect this is wrong for 3.6 and 3.5, but
# I haven't verified what the 3.7ish fix is # I haven't verified what the 3.7ish fix is
elif opname == 'BUILD_MAP_UNPACK_WITH_CALL': elif opname == "BUILD_MAP_UNPACK_WITH_CALL":
if self.version < (3, 7): if self.version < (3, 7):
self.addRule("expr ::= unmapexpr", nop_func) self.addRule("expr ::= unmapexpr", nop_func)
nargs = token.attr % 256 nargs = token.attr % 256
map_unpack_n = "map_unpack_%s" % nargs map_unpack_n = "map_unpack_%s" % nargs
rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs) rule = map_unpack_n + " ::= " + "expr " * (nargs)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname) rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
call_token = tokens[i+1] call_token = tokens[i + 1]
rule = 'call ::= expr unmapexpr ' + call_token.kind rule = "call ::= expr unmapexpr " + call_token.kind
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH' and self.version < (3, 8): elif opname == "BEFORE_ASYNC_WITH" and self.version < (3, 8):
# Some Python 3.5+ async additions # Some Python 3.5+ async additions
rules_str = """ rules_str = """
stmt ::= async_with_stmt stmt ::= async_with_stmt
@@ -199,24 +203,27 @@ class Python35Parser(Python34Parser):
async_with_post async_with_post
""" """
self.addRule(rules_str, nop_func) self.addRule(rules_str, nop_func)
elif opname == 'BUILD_MAP_UNPACK': elif opname == "BUILD_MAP_UNPACK":
self.addRule(""" self.addRule(
"""
expr ::= dict_unpack expr ::= dict_unpack
dict_unpack ::= dict_comp BUILD_MAP_UNPACK dict_unpack ::= dict_comp BUILD_MAP_UNPACK
""", nop_func) """,
nop_func,
)
elif opname == 'SETUP_WITH': elif opname == "SETUP_WITH":
# Python 3.5+ has WITH_CLEANUP_START/FINISH # Python 3.5+ has WITH_CLEANUP_START/FINISH
rules_str = """ rules_str = """
with ::= expr with ::= expr
SETUP_WITH POP_TOP suite_stmts_opt SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
withasstmt ::= expr with_as ::= expr
SETUP_WITH store suite_stmts_opt SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
""" """
self.addRule(rules_str, nop_func) self.addRule(rules_str, nop_func)
pass pass
@@ -230,19 +237,24 @@ class Python35Parser(Python34Parser):
# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
# 2 for * and ** args (CALL_FUNCTION_VAR_KW). # 2 for * and ** args (CALL_FUNCTION_VAR_KW).
# Yes, this computation based on instruction name is a little bit hoaky. # Yes, this computation based on instruction name is a little bit hoaky.
nak = ( len(opname)-len('CALL_FUNCTION') ) // 3 nak = (len(opname) - len("CALL_FUNCTION")) // 3
uniq_param = args_kw + args_pos uniq_param = args_kw + args_pos
if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops): if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
rule = ('async_call ::= expr ' + rule = (
('pos_arg ' * args_pos) + "async_call ::= expr "
('kwarg ' * args_kw) + + ("pos_arg " * args_pos)
'expr ' * nak + token.kind + + ("kwarg " * args_kw)
' GET_AWAITABLE LOAD_CONST YIELD_FROM') + "expr " * nak
+ token.kind
+ " GET_AWAITABLE LOAD_CONST YIELD_FROM"
)
self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule(rule, token.kind, uniq_param, customize)
self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize) self.add_unique_rule(
"expr ::= async_call", token.kind, uniq_param, customize
)
if opname.startswith('CALL_FUNCTION_VAR'): if opname.startswith("CALL_FUNCTION_VAR"):
# Python 3.5 changes the stack position of *args. KW args come # Python 3.5 changes the stack position of *args. KW args come
# after *args. # after *args.
@@ -250,43 +262,55 @@ class Python35Parser(Python34Parser):
# CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX # CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
token.kind = self.call_fn_name(token) token.kind = self.call_fn_name(token)
if opname.endswith('KW'): if opname.endswith("KW"):
kw = 'expr ' kw = "expr "
else: else:
kw = '' kw = ""
rule = ('call ::= expr expr ' + rule = (
('pos_arg ' * args_pos) + "call ::= expr expr "
('kwarg ' * args_kw) + kw + token.kind) + ("pos_arg " * args_pos)
+ ("kwarg " * args_kw)
+ kw
+ token.kind
)
# Note: semantic actions make use of the fact of whether "args_pos" # Note: semantic actions make use of the fact of whether "args_pos"
# zero or not in creating a template rule. # zero or not in creating a template rule.
self.add_unique_rule(rule, token.kind, args_pos, customize) self.add_unique_rule(rule, token.kind, args_pos, customize)
else: else:
super(Python35Parser, self).custom_classfunc_rule(opname, token, customize, *args super(Python35Parser, self).custom_classfunc_rule(
opname, token, customize, *args
) )
class Python35ParserSingle(Python35Parser, PythonParserSingle): class Python35ParserSingle(Python35Parser, PythonParserSingle):
pass pass
if __name__ == '__main__':
if __name__ == "__main__":
# Check grammar # Check grammar
p = Python35Parser() p = Python35Parser()
p.check_grammar() p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 5): if PYTHON_VERSION_TRIPLE[:2] == (3, 5):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY) s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
opcode_set = set(s.opc.opname).union(set( opcode_set = set(s.opc.opname).union(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST LAMBDA_MARKER RETURN_LAST
""".split())) """.split()
)
)
remain_tokens = set(tokens) - opcode_set remain_tokens = set(tokens) - opcode_set
import re import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens) print(remain_tokens)
# print(sorted(p.rule2name.items())) # print(sorted(p.rule2name.items()))

217
uncompyle6/parsers/parse36.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2020, 2022-2023 Rocky Bernstein # Copyright (c) 2016-2020, 2022-2024 Rocky Bernstein
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@@ -17,24 +17,25 @@ spark grammar differences over Python 3.5 for Python 3.6.
""" """
from __future__ import print_function from __future__ import print_function
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse35 import Python35Parser from uncompyle6.parsers.parse35 import Python35Parser
from uncompyle6.scanners.tok import Token from uncompyle6.scanners.tok import Token
class Python36Parser(Python35Parser):
class Python36Parser(Python35Parser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python36Parser, self).__init__(debug_parser) super(Python36Parser, self).__init__(debug_parser)
self.customized = {} self.customized = {}
def p_36_jump(self, args): def p_36_jump(self, args):
""" """
# Zero or one COME_FROM # Zero or one COME_FROM
# And/or expressions have this # And/or expressions have this
come_from_opt ::= COME_FROM? come_from_opt ::= COME_FROM?
""" """
def p_36_misc(self, args): def p_36_misc(self, args):
"""sstmt ::= sstmt RETURN_LAST """sstmt ::= sstmt RETURN_LAST
@@ -207,7 +208,8 @@ class Python36Parser(Python35Parser):
# self.remove_rules(""" # self.remove_rules("""
# """) # """)
super(Python36Parser, self).customize_grammar_rules(tokens, customize) super(Python36Parser, self).customize_grammar_rules(tokens, customize)
self.remove_rules(""" self.remove_rules(
"""
_ifstmts_jumpl ::= c_stmts_opt _ifstmts_jumpl ::= c_stmts_opt
_ifstmts_jumpl ::= _ifstmts_jump _ifstmts_jumpl ::= _ifstmts_jump
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM
@@ -234,7 +236,8 @@ class Python36Parser(Python35Parser):
for_block pb_ja for_block pb_ja
else_suite COME_FROM_LOOP else_suite COME_FROM_LOOP
""") """
)
self.check_reduce["call_kw"] = "AST" self.check_reduce["call_kw"] = "AST"
# Opcode names in the custom_ops_processed set have rules that get added # Opcode names in the custom_ops_processed set have rules that get added
@@ -247,24 +250,23 @@ class Python36Parser(Python35Parser):
# the start. # the start.
custom_ops_processed = set() custom_ops_processed = set()
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
opname = token.kind opname = token.kind
if opname == 'FORMAT_VALUE': if opname == "FORMAT_VALUE":
rules_str = """ rules_str = """
expr ::= formatted_value1 expr ::= formatted_value1
formatted_value1 ::= expr FORMAT_VALUE formatted_value1 ::= expr FORMAT_VALUE
""" """
self.add_unique_doc_rules(rules_str, customize) self.add_unique_doc_rules(rules_str, customize)
elif opname == 'FORMAT_VALUE_ATTR': elif opname == "FORMAT_VALUE_ATTR":
rules_str = """ rules_str = """
expr ::= formatted_value2 expr ::= formatted_value2
formatted_value2 ::= expr expr FORMAT_VALUE_ATTR formatted_value2 ::= expr expr FORMAT_VALUE_ATTR
""" """
self.add_unique_doc_rules(rules_str, customize) self.add_unique_doc_rules(rules_str, customize)
elif opname == 'MAKE_FUNCTION_CLOSURE': elif opname == "MAKE_FUNCTION_CLOSURE":
if 'LOAD_DICTCOMP' in self.seen_ops: if "LOAD_DICTCOMP" in self.seen_ops:
# Is there something general going on here? # Is there something general going on here?
rule = """ rule = """
dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
@@ -272,7 +274,7 @@ class Python36Parser(Python35Parser):
GET_ITER CALL_FUNCTION_1 GET_ITER CALL_FUNCTION_1
""" """
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif 'LOAD_SETCOMP' in self.seen_ops: elif "LOAD_SETCOMP" in self.seen_ops:
rule = """ rule = """
set_comp ::= load_closure LOAD_SETCOMP LOAD_STR set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
MAKE_FUNCTION_CLOSURE expr MAKE_FUNCTION_CLOSURE expr
@@ -280,7 +282,7 @@ class Python36Parser(Python35Parser):
""" """
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH': elif opname == "BEFORE_ASYNC_WITH":
rules_str = """ rules_str = """
stmt ::= async_with_stmt stmt ::= async_with_stmt
async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
@@ -306,30 +308,37 @@ class Python36Parser(Python35Parser):
""" """
self.addRule(rules_str, nop_func) self.addRule(rules_str, nop_func)
elif opname.startswith('BUILD_STRING'): elif opname.startswith("BUILD_STRING"):
v = token.attr v = token.attr
rules_str = """ rules_str = """
expr ::= joined_str expr ::= joined_str
joined_str ::= %sBUILD_STRING_%d joined_str ::= %sBUILD_STRING_%d
""" % ("expr " * v, v) """ % (
"expr " * v,
v,
)
self.add_unique_doc_rules(rules_str, customize) self.add_unique_doc_rules(rules_str, customize)
if 'FORMAT_VALUE_ATTR' in self.seen_ops: if "FORMAT_VALUE_ATTR" in self.seen_ops:
rules_str = """ rules_str = """
formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
expr ::= formatted_value_attr expr ::= formatted_value_attr
""" """
self.add_unique_doc_rules(rules_str, customize) self.add_unique_doc_rules(rules_str, customize)
elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'): elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
v = token.attr v = token.attr
rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname) rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'): elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
v = token.attr v = token.attr
rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) + rule = (
'expr32 ' * int((v//32) % 32) + "build_tuple_unpack_with_call ::= "
'expr ' * (v % 32) + opname) + "expr1024 " * int(v // 1024)
+ "expr32 " * int((v // 32) % 32)
+ "expr " * (v % 32)
+ opname
)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
rule = ('starred ::= %s %s' % ('expr ' * v, opname)) rule = "starred ::= %s %s" % ("expr " * v, opname)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname == "GET_AITER": elif opname == "GET_AITER":
self.addRule( self.addRule(
@@ -475,7 +484,6 @@ class Python36Parser(Python35Parser):
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == "GET_ANEXT": elif opname == "GET_ANEXT":
self.addRule( self.addRule(
""" """
@@ -500,7 +508,7 @@ class Python36Parser(Python35Parser):
) )
custom_ops_processed.add(opname) custom_ops_processed.add(opname)
elif opname == 'SETUP_ANNOTATIONS': elif opname == "SETUP_ANNOTATIONS":
# 3.6 Variable Annotations PEP 526 # 3.6 Variable Annotations PEP 526
# This seems to come before STORE_ANNOTATION, and doesn't # This seems to come before STORE_ANNOTATION, and doesn't
# correspond to direct Python source code. # correspond to direct Python source code.
@@ -516,7 +524,7 @@ class Python36Parser(Python35Parser):
""" """
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
# Check to combine assignment + annotation into one statement # Check to combine assignment + annotation into one statement
self.check_reduce['assign'] = 'token' self.check_reduce["assign"] = "token"
elif opname == "WITH_CLEANUP_START": elif opname == "WITH_CLEANUP_START":
rules_str = """ rules_str = """
stmt ::= with_null stmt ::= with_null
@@ -524,13 +532,13 @@ class Python36Parser(Python35Parser):
with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
""" """
self.addRule(rules_str, nop_func) self.addRule(rules_str, nop_func)
elif opname == 'SETUP_WITH': elif opname == "SETUP_WITH":
rules_str = """ rules_str = """
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH with ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
with_suffix with_suffix
# Removes POP_BLOCK LOAD_CONST from 3.6- # Removes POP_BLOCK LOAD_CONST from 3.6-
withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_suffix with_suffix
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_WITH BEGIN_FINALLY COME_FROM_WITH
@@ -542,7 +550,6 @@ class Python36Parser(Python35Parser):
return return
def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy): def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
# Additional exprs for * and ** args: # Additional exprs for * and ** args:
@@ -550,140 +557,186 @@ class Python36Parser(Python35Parser):
# 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW # 1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
# 2 for * and ** args (CALL_FUNCTION_VAR_KW). # 2 for * and ** args (CALL_FUNCTION_VAR_KW).
# Yes, this computation based on instruction name is a little bit hoaky. # Yes, this computation based on instruction name is a little bit hoaky.
nak = ( len(opname)-len('CALL_FUNCTION') ) // 3 nak = (len(opname) - len("CALL_FUNCTION")) // 3
uniq_param = args_kw + args_pos uniq_param = args_kw + args_pos
if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops): if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
rule = ('async_call ::= expr ' + rule = (
('pos_arg ' * args_pos) + "async_call ::= expr "
('kwarg ' * args_kw) + + ("pos_arg " * args_pos)
'expr ' * nak + token.kind + + ("kwarg " * args_kw)
' GET_AWAITABLE LOAD_CONST YIELD_FROM') + "expr " * nak
+ token.kind
+ " GET_AWAITABLE LOAD_CONST YIELD_FROM"
)
self.add_unique_rule(rule, token.kind, uniq_param, customize) self.add_unique_rule(rule, token.kind, uniq_param, customize)
self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize) self.add_unique_rule(
"expr ::= async_call", token.kind, uniq_param, customize
)
if opname.startswith('CALL_FUNCTION_KW'): if opname.startswith("CALL_FUNCTION_KW"):
if is_pypy: if is_pypy:
# PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions # PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions
super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy) super(Python36Parser, self).custom_classfunc_rule(
opname, token, customize, next_token, is_pypy
)
else: else:
self.addRule("expr ::= call_kw36", nop_func) self.addRule("expr ::= call_kw36", nop_func)
values = 'expr ' * token.attr values = "expr " * token.attr
rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(
**locals()
)
self.add_unique_rule(rule, token.kind, token.attr, customize) self.add_unique_rule(rule, token.kind, token.attr, customize)
elif opname == 'CALL_FUNCTION_EX_KW': elif opname == "CALL_FUNCTION_EX_KW":
# Note: this doesn't exist in 3.7 and later # Note: this doesn't exist in 3.7 and later
self.addRule("""expr ::= call_ex_kw4 self.addRule(
"""expr ::= call_ex_kw4
call_ex_kw4 ::= expr call_ex_kw4 ::= expr
expr expr
expr expr
CALL_FUNCTION_EX_KW CALL_FUNCTION_EX_KW
""", """,
nop_func) nop_func,
if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_op_basenames: )
self.addRule("""expr ::= call_ex_kw if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
self.addRule(
"""expr ::= call_ex_kw
call_ex_kw ::= expr expr build_map_unpack_with_call call_ex_kw ::= expr expr build_map_unpack_with_call
CALL_FUNCTION_EX_KW CALL_FUNCTION_EX_KW
""", nop_func) """,
if 'BUILD_TUPLE_UNPACK_WITH_CALL' in self.seen_op_basenames: nop_func,
)
if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
# FIXME: should this be parameterized by EX value? # FIXME: should this be parameterized by EX value?
self.addRule("""expr ::= call_ex_kw3 self.addRule(
"""expr ::= call_ex_kw3
call_ex_kw3 ::= expr call_ex_kw3 ::= expr
build_tuple_unpack_with_call build_tuple_unpack_with_call
expr expr
CALL_FUNCTION_EX_KW CALL_FUNCTION_EX_KW
""", nop_func) """,
if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_op_basenames: nop_func,
)
if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
# FIXME: should this be parameterized by EX value? # FIXME: should this be parameterized by EX value?
self.addRule("""expr ::= call_ex_kw2 self.addRule(
"""expr ::= call_ex_kw2
call_ex_kw2 ::= expr call_ex_kw2 ::= expr
build_tuple_unpack_with_call build_tuple_unpack_with_call
build_map_unpack_with_call build_map_unpack_with_call
CALL_FUNCTION_EX_KW CALL_FUNCTION_EX_KW
""", nop_func) """,
nop_func,
)
elif opname == 'CALL_FUNCTION_EX': elif opname == "CALL_FUNCTION_EX":
self.addRule(""" self.addRule(
"""
expr ::= call_ex expr ::= call_ex
starred ::= expr starred ::= expr
call_ex ::= expr starred CALL_FUNCTION_EX call_ex ::= expr starred CALL_FUNCTION_EX
""", nop_func) """,
nop_func,
)
if self.version >= (3, 6): if self.version >= (3, 6):
if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_ops: if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
self.addRule(""" self.addRule(
"""
expr ::= call_ex_kw expr ::= call_ex_kw
call_ex_kw ::= expr expr call_ex_kw ::= expr expr
build_map_unpack_with_call CALL_FUNCTION_EX build_map_unpack_with_call CALL_FUNCTION_EX
""", nop_func) """,
if 'BUILD_TUPLE_UNPACK_WITH_CALL' in self.seen_ops: nop_func,
self.addRule(""" )
if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
self.addRule(
"""
expr ::= call_ex_kw3 expr ::= call_ex_kw3
call_ex_kw3 ::= expr call_ex_kw3 ::= expr
build_tuple_unpack_with_call build_tuple_unpack_with_call
%s %s
CALL_FUNCTION_EX CALL_FUNCTION_EX
""" % 'expr ' * token.attr, nop_func) """
% "expr "
* token.attr,
nop_func,
)
pass pass
# FIXME: Is this right? # FIXME: Is this right?
self.addRule(""" self.addRule(
"""
expr ::= call_ex_kw4 expr ::= call_ex_kw4
call_ex_kw4 ::= expr call_ex_kw4 ::= expr
expr expr
expr expr
CALL_FUNCTION_EX CALL_FUNCTION_EX
""", nop_func) """,
nop_func,
)
pass pass
else: else:
super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy) super(Python36Parser, self).custom_classfunc_rule(
opname, token, customize, next_token, is_pypy
)
def reduce_is_invalid(self, rule, ast, tokens, first, last): def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python36Parser, invalid = super(Python36Parser, self).reduce_is_invalid(
self).reduce_is_invalid(rule, ast, rule, ast, tokens, first, last
tokens, first, last) )
if invalid: if invalid:
return invalid return invalid
if rule[0] == 'assign': if rule[0] == "assign":
# Try to combine assignment + annotation into one statement # Try to combine assignment + annotation into one statement
if (len(tokens) >= last + 1 and if (
tokens[last] == 'LOAD_NAME' and len(tokens) >= last + 1
tokens[last+1] == 'STORE_ANNOTATION' and and tokens[last] == "LOAD_NAME"
tokens[last-1].pattr == tokens[last+1].pattr): and tokens[last + 1] == "STORE_ANNOTATION"
and tokens[last - 1].pattr == tokens[last + 1].pattr
):
# Will handle as ann_assign_init_value # Will handle as ann_assign_init_value
return True return True
pass pass
if rule[0] == 'call_kw': if rule[0] == "call_kw":
# Make sure we don't derive call_kw # Make sure we don't derive call_kw
nt = ast[0] nt = ast[0]
while not isinstance(nt, Token): while not isinstance(nt, Token):
if nt[0] == 'call_kw': if nt[0] == "call_kw":
return True return True
nt = nt[0] nt = nt[0]
pass pass
pass pass
return False return False
class Python36ParserSingle(Python36Parser, PythonParserSingle): class Python36ParserSingle(Python36Parser, PythonParserSingle):
pass pass
if __name__ == '__main__':
if __name__ == "__main__":
# Check grammar # Check grammar
p = Python36Parser() p = Python36Parser()
p.check_grammar() p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 6): if PYTHON_VERSION_TRIPLE[:2] == (3, 6):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY) s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
opcode_set = set(s.opc.opname).union(set( opcode_set = set(s.opc.opname).union(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST LAMBDA_MARKER RETURN_LAST
""".split())) """.split()
)
)
remain_tokens = set(tokens) - opcode_set remain_tokens = set(tokens) - opcode_set
import re import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens) print(remain_tokens)
# print(sorted(p.rule2name.items())) # print(sorted(p.rule2name.items()))

19
uncompyle6/parsers/parse37.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2020, 2022-2023 Rocky Bernstein # Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
@@ -17,10 +17,12 @@ Python 3.7 grammar for the spark Earley-algorithm parser.
""" """
from __future__ import print_function from __future__ import print_function
from uncompyle6.scanners.tok import Token
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse37base import Python37BaseParser from uncompyle6.parsers.parse37base import Python37BaseParser
from uncompyle6.scanners.tok import Token
class Python37Parser(Python37BaseParser): class Python37Parser(Python37BaseParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -249,8 +251,7 @@ class Python37Parser(Python37BaseParser):
""" """
def p_generator_exp(self, args): def p_generator_exp(self, args):
""" """ """
"""
def p_jump(self, args): def p_jump(self, args):
""" """
@@ -757,7 +758,7 @@ class Python37Parser(Python37BaseParser):
""" """
def p_dict_comp3(self, args): def p_dict_comp3(self, args):
"""" """ "
expr ::= dict_comp expr ::= dict_comp
stmt ::= dict_comp_func stmt ::= dict_comp_func
@@ -1554,7 +1555,7 @@ class Python37Parser(Python37BaseParser):
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
# Removes POP_BLOCK LOAD_CONST from 3.6- # Removes POP_BLOCK LOAD_CONST from 3.6-
withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
""" """
if self.version < (3, 8): if self.version < (3, 8):
@@ -1575,7 +1576,6 @@ class Python37Parser(Python37BaseParser):
pass pass
def custom_classfunc_rule(self, opname, token, customize, next_token): def custom_classfunc_rule(self, opname, token, customize, next_token):
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
# Additional exprs for * and ** args: # Additional exprs for * and ** args:
@@ -1718,6 +1718,7 @@ class Python37Parser(Python37BaseParser):
pass pass
return False return False
def info(args): def info(args):
# Check grammar # Check grammar
p = Python37Parser() p = Python37Parser()
@@ -1748,7 +1749,7 @@ if __name__ == "__main__":
# FIXME: DRY this with other parseXX.py routines # FIXME: DRY this with other parseXX.py routines
p = Python37Parser() p = Python37Parser()
p.check_grammar() p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 7): if PYTHON_VERSION_TRIPLE[:2] == (3, 7):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()

25
uncompyle6/parsers/parse37base.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2017, 2019-2020, 2022-2023 Rocky Bernstein # Copyright (c) 2016-2017, 2019-2020, 2022-2024 Rocky Bernstein
""" """
Python 3.7 base code. We keep non-custom-generated grammar rules out of this file. Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
""" """
@@ -1055,14 +1055,14 @@ class Python37BaseParser(PythonParser):
elif opname == "SETUP_WITH": elif opname == "SETUP_WITH":
rules_str = """ rules_str = """
stmt ::= with stmt ::= with
stmt ::= withasstmt stmt ::= with_as
with ::= expr with ::= expr
SETUP_WITH POP_TOP SETUP_WITH POP_TOP
suite_stmts_opt suite_stmts_opt
COME_FROM_WITH COME_FROM_WITH
with_suffix with_suffix
withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
with_suffix with_suffix
with ::= expr with ::= expr
@@ -1071,7 +1071,7 @@ class Python37BaseParser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix with_suffix
withasstmt ::= expr with_as ::= expr
SETUP_WITH store suite_stmts_opt SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix with_suffix
@@ -1080,7 +1080,7 @@ class Python37BaseParser(PythonParser):
SETUP_WITH POP_TOP suite_stmts_opt SETUP_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix with_suffix
withasstmt ::= expr with_as ::= expr
SETUP_WITH store suite_stmts_opt SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix with_suffix
@@ -1098,17 +1098,18 @@ class Python37BaseParser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH POP_BLOCK LOAD_CONST COME_FROM_WITH
with_suffix with_suffix
withasstmt ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
withasstmt ::= expr
SETUP_WITH store suite_stmts
POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK with ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_WITH BEGIN_FINALLY COME_FROM_WITH
with_suffix with_suffix
with_as ::= expr
SETUP_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_WITH
with_as ::= expr
SETUP_WITH store suite_stmts
POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
""" """
self.addRule(rules_str, nop_func) self.addRule(rules_str, nop_func)

135
uncompyle6/semantics/consts.py
View File

@@ -378,6 +378,46 @@ TABLE_DIRECT = {
(0, PRECEDENCE["named_expr"]-1)), (0, PRECEDENCE["named_expr"]-1)),
"break": ("%|break\n",), "break": ("%|break\n",),
"continue": ("%|continue\n",), "continue": ("%|continue\n",),
"except": ("%|except:\n%+%c%-", 3),
"except_cond1": ("%|except %c:\n", 1),
"except_cond2": ("%|except %c as %c:\n", (1, "expr"), (5, "store")),
"except_suite": ("%+%c%-%C", 0, (1, maxint, "")),
# In Python 3.6+, this is more complicated in the presence of "returns"
"except_suite_finalize": ("%+%c%-%C", 1, (3, maxint, "")),
"expr_stmt": (
"%|%p\n",
# When a statement contains only a named_expr (:=)
# the named_expr should have parenthesis around it.
(0, "expr", PRECEDENCE["named_expr"] - 1)
),
# Note: Python 3.8+ changes this
"for": ("%|for %c in %c:\n%+%c%-\n\n", (3, "store"), (1, "expr"), (4, "for_block")),
"forelsestmt": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(3, "store"),
(1, "expr"),
(4, "for_block"),
-2,
),
"forelselaststmt": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-",
(3, "store"),
(1, "expr"),
(4, "for_block"),
-2,
),
"forelselaststmtl": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(3, "store"),
(1, "expr"),
(4, "for_block"),
-2,
),
"raise_stmt0": ("%|raise\n",), "raise_stmt0": ("%|raise\n",),
"raise_stmt1": ("%|raise %c\n", 0), "raise_stmt1": ("%|raise %c\n", 0),
"raise_stmt3": ("%|raise %c, %c, %c\n", 0, 1, 2), "raise_stmt3": ("%|raise %c, %c, %c\n", 0, 1, 2),
@@ -419,72 +459,7 @@ TABLE_DIRECT = {
1, 1,
3, 3,
), # has COME_FROM ), # has COME_FROM
"whileTruestmt": ("%|while True:\n%+%c%-\n\n", 1),
"whilestmt": ("%|while %c:\n%+%c%-\n\n", 1, 2),
"while1stmt": ("%|while 1:\n%+%c%-\n\n", 1),
"while1elsestmt": ("%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n", 1, -2),
"whileelsestmt": ("%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n", 1, 2, -2),
"whileelsestmt2": ("%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n", 1, 2, -3),
"whileelselaststmt": ("%|while %c:\n%+%c%-%|else:\n%+%c%-", 1, 2, -2),
# If there are situations where we need "with ... as ()"
# We may need to customize this in n_withasstmt
"withasstmt": (
"%|with %c as %c:\n%+%c%-",
(0, "expr"),
(2, "store"),
(3, ("suite_stmts_opt", "suite_stmts")),
),
"expr_stmt": (
"%|%p\n",
# When a statement contains only a named_expr (:=)
# the named_expr should have parenthesis around it.
(0, "expr", PRECEDENCE["named_expr"] - 1)
),
# Note: Python 3.8+ changes this
"for": ("%|for %c in %c:\n%+%c%-\n\n", (3, "store"), (1, "expr"), (4, "for_block")),
"forelsestmt": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(3, "store"),
(1, "expr"),
(4, "for_block"),
-2,
),
"forelselaststmt": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-",
(3, "store"),
(1, "expr"),
(4, "for_block"),
-2,
),
"forelselaststmtl": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(3, "store"),
(1, "expr"),
(4, "for_block"),
-2,
),
"try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3),
"tryelsestmt": ("%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n", 1, 3, 4),
"tryelsestmtc": ("%|try:\n%+%c%-%c%|else:\n%+%c%-", 1, 3, 4),
"tryelsestmtl": ("%|try:\n%+%c%-%c%|else:\n%+%c%-", 1, 3, 4),
# Note: this is generated generated by grammar rules but in this phase.
"tf_try_except": ("%c%-%c%+", 1, 3),
"tf_tryelsestmt": ("%c%-%c%|else:\n%+%c", 1, 3, 4),
"tryfinallystmt": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", 1, 5),
"except": ("%|except:\n%+%c%-", 3),
"except_cond1": ("%|except %c:\n", 1),
"except_cond2": ("%|except %c as %c:\n", (1, "expr"), (5, "store")),
"except_suite": ("%+%c%-%C", 0, (1, maxint, "")),
# In Python 3.6+, this is more complicated in the presence of "returns"
"except_suite_finalize": ("%+%c%-%C", 1, (3, maxint, "")),
"pass": ("%|pass\n",),
"STORE_FAST": ("%{pattr}",),
"kv": ("%c: %c", 3, 1),
"kv2": ("%c: %c", 1, 2),
"import": ("%|import %c\n", 2), "import": ("%|import %c\n", 2),
"importlist": ("%C", (0, maxint, ", ")), "importlist": ("%C", (0, maxint, ", ")),
@@ -498,6 +473,36 @@ TABLE_DIRECT = {
"import_from_star": ( "import_from_star": (
"%|from %[2]{pattr} import *\n", "%|from %[2]{pattr} import *\n",
), ),
"kv": ("%c: %c", 3, 1),
"kv2": ("%c: %c", 1, 2),
"pass": ("%|pass\n",),
"whileTruestmt": ("%|while True:\n%+%c%-\n\n", 1),
"whilestmt": ("%|while %c:\n%+%c%-\n\n", 1, 2),
"while1stmt": ("%|while 1:\n%+%c%-\n\n", 1),
"while1elsestmt": ("%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n", 1, -2),
"whileelsestmt": ("%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n", 1, 2, -2),
"whileelsestmt2": ("%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n", 1, 2, -3),
"whileelselaststmt": ("%|while %c:\n%+%c%-%|else:\n%+%c%-", 1, 2, -2),
# If there are situations where we need "with ... as ()"
# We may need to customize this in n_with_as
"with_as": (
"%|with %c as %c:\n%+%c%-",
(0, "expr"),
(2, "store"),
(3, ("suite_stmts_opt", "suite_stmts")),
),
"try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3),
"tryelsestmt": ("%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n", 1, 3, 4),
"tryelsestmtc": ("%|try:\n%+%c%-%c%|else:\n%+%c%-", 1, 3, 4),
"tryelsestmtl": ("%|try:\n%+%c%-%c%|else:\n%+%c%-", 1, 3, 4),
# Note: this is generated generated by grammar rules but in this phase.
"tf_try_except": ("%c%-%c%+", 1, 3),
"tf_tryelsestmt": ("%c%-%c%|else:\n%+%c", 1, 3, 4),
"tryfinallystmt": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", 1, 5),
} }