jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 01:09:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

DRY code; localize 3.{5,6} grammar rules..

Browse Source 
helper.py, pysource: has code to flatten list used in n_list and n_build_tuple_unpack_with_call
parse3{5,6}.py: localize grammar rules with BEFORE_ASYNC_WITH
This commit is contained in:
rocky
2017-12-11 21:26:58 -05:00
parent 3705f6d096
commit b84c35acf5
5 changed files with 96 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pytest/test_grammar.py
View File

@@ -35,6 +35,7 @@ def test_grammar():
expect_lhs.add("annotate_arg") expect_lhs.add("annotate_arg")
expect_lhs.add("annotate_tuple") expect_lhs.add("annotate_tuple")
unused_rhs.add("mkfunc_annotate") unused_rhs.add("mkfunc_annotate")
unused_rhs.add('call')
if PYTHON_VERSION < 3.6: if PYTHON_VERSION < 3.6:
# 3.6 has at least one non-custom call rule # 3.6 has at least one non-custom call rule
# the others don't # the others don't
@@ -47,8 +48,6 @@ def test_grammar():
else: else:
expect_right_recursive.add((('l_stmts', expect_right_recursive.add((('l_stmts',
('lastl_stmt', 'COME_FROM', 'l_stmts')))) ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
unused_rhs.add('build_map_unpack_with_call')
unused_rhs.add('unmapexpr')
# expect_lhs.add('kwargs1') # expect_lhs.add('kwargs1')
pass pass
pass pass

44
uncompyle6/parsers/parse35.py
View File

@@ -57,24 +57,6 @@ class Python35Parser(Python34Parser):
# Python 3.5+ async additions # Python 3.5+ async additions
stmt ::= async_with_stmt
async_with_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
stmt ::= async_with_as_stmt
async_with_as_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
stmt ::= async_for_stmt stmt ::= async_for_stmt
async_for_stmt ::= SETUP_LOOP expr async_for_stmt ::= SETUP_LOOP expr
GET_AITER GET_AITER
@@ -153,14 +135,36 @@ class Python35Parser(Python34Parser):
nargs = token.attr % 256 nargs = token.attr % 256
map_unpack_n = "map_unpack_%s" % nargs map_unpack_n = "map_unpack_%s" % nargs
rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs) rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs)
print("XXXX", rule)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname) rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
call_token = tokens[i+1] call_token = tokens[i+1]
rule = 'call ::= expr unmapexpr ' + call_token.kind rule = 'call ::= expr unmapexpr ' + call_token.kind
print("XXXX2", rule)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname == 'BEFORE_ASYNC_WITH':
rules_str = """
async_with_stmt ::= expr
stmt ::= async_with_stmt
async_with_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
stmt ::= async_with_as_stmt
async_with_as_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH store suite_stmts_opt
POP_BLOCK LOAD_CONST
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
elif opname == 'BUILD_MAP_UNPACK': elif opname == 'BUILD_MAP_UNPACK':
self.addRule(""" self.addRule("""
expr ::= unmap_dict expr ::= unmap_dict

61
uncompyle6/parsers/parse36.py
View File

@@ -39,23 +39,6 @@ class Python36Parser(Python35Parser):
# Adds a COME_FROM_ASYNC_WITH over 3.5 # Adds a COME_FROM_ASYNC_WITH over 3.5
# FIXME: remove corresponding rule for 3.5? # FIXME: remove corresponding rule for 3.5?
async_with_as_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH store
suite_stmts_opt
POP_BLOCK LOAD_CONST
COME_FROM_ASYNC_WITH
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
async_with_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST
COME_FROM_ASYNC_WITH
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM
@@ -83,6 +66,30 @@ class Python36Parser(Python35Parser):
fstring_single ::= expr FORMAT_VALUE fstring_single ::= expr FORMAT_VALUE
""" """
self.add_unique_doc_rules(rules_str, customize) self.add_unique_doc_rules(rules_str, customize)
elif opname == 'BEFORE_ASYNC_WITH':
rules_str = """
stmt ::= async_with_stmt
async_with_as_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH store
suite_stmts_opt
POP_BLOCK LOAD_CONST
COME_FROM_ASYNC_WITH
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
stmt ::= async_with_as_stmt
async_with_stmt ::= expr
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
POP_BLOCK LOAD_CONST
COME_FROM_ASYNC_WITH
WITH_CLEANUP_START
GET_AWAITABLE LOAD_CONST YIELD_FROM
WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
elif opname == 'BUILD_STRING': elif opname == 'BUILD_STRING':
v = token.attr v = token.attr
joined_str_n = "formatted_value_%s" % v joined_str_n = "formatted_value_%s" % v
@@ -117,18 +124,6 @@ class Python36Parser(Python35Parser):
'expr32 ' * int((v//32) % 32) + 'expr32 ' * int((v//32) % 32) +
'expr ' * (v % 32) + opname) 'expr ' * (v % 32) + opname)
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname.startswith('CALL_FUNCTION_EX'):
self.addRule("expr ::= call_ex", nop_func)
if token.attr == 0:
self.addRule("""
unpack_list ::= list
call_ex ::= expr unpack_list CALL_FUNCTION_EX
""", nop_func)
else:
self.addRule(
'call_ex ::= expr build_tuple_unpack_with_call CALL_FUNCTION_EX',
nop_func)
pass
elif opname == 'CALL_FUNCTION_EX_KW': elif opname == 'CALL_FUNCTION_EX_KW':
args_pos, args_kw = self.get_pos_kw(token) args_pos, args_kw = self.get_pos_kw(token)
uniq_param = args_kw + args_pos uniq_param = args_kw + args_pos
@@ -137,6 +132,13 @@ class Python36Parser(Python35Parser):
'expr build_tuple_unpack_with_call build_map_unpack_with_call ' 'expr build_tuple_unpack_with_call build_map_unpack_with_call '
'CALL_FUNCTION_EX_KW') 'CALL_FUNCTION_EX_KW')
self.addRule(rule, nop_func) self.addRule(rule, nop_func)
elif opname == 'CALL_FUNCTION_EX':
self.addRule("expr ::= call_ex", nop_func)
self.addRule("""
unpack_list ::= list
call_ex ::= expr unpack_list CALL_FUNCTION_EX
""", nop_func)
pass
else: else:
super(Python36Parser, self).custom_classfunc_rule(opname, token, super(Python36Parser, self).custom_classfunc_rule(opname, token,
customize, customize,
@@ -144,7 +146,6 @@ class Python36Parser(Python35Parser):
seen_GET_AWAITABLE_YIELD_FROM, seen_GET_AWAITABLE_YIELD_FROM,
next_token) next_token)
class Python36ParserSingle(Python36Parser, PythonParserSingle): class Python36ParserSingle(Python36Parser, PythonParserSingle):
pass pass

22
uncompyle6/semantics/helper.py
View File

@@ -127,6 +127,28 @@ def print_docstring(self, indent, docstring):
self.println(indent, trimmed[-1], quote) self.println(indent, trimmed[-1], quote)
return True return True
def flatten_list(node):
"""
List of expressions may be nested in groups of 32 and 1024
items. flatten that out and return the list
"""
flat_elems = []
for elem in node:
if elem == 'expr1024':
for subelem in elem:
for subsubelem in subelem:
flat_elems.append(subsubelem)
elif elem == 'expr32':
for subelem in elem:
flat_elems.append(subelem)
else:
flat_elems.append(elem)
pass
pass
return flat_elems
# if __name__ == '__main__': # if __name__ == '__main__':
# if PYTHON3: # if PYTHON3:
# from io import StringIO # from io import StringIO

56
uncompyle6/semantics/pysource.py
View File

@@ -127,7 +127,7 @@ from uncompyle6.semantics.make_function import (
from uncompyle6.semantics.parser_error import ParserError from uncompyle6.semantics.parser_error import ParserError
from uncompyle6.semantics.check_ast import checker from uncompyle6.semantics.check_ast import checker
from uncompyle6.semantics.helper import ( from uncompyle6.semantics.helper import (
print_docstring, find_globals) print_docstring, find_globals, flatten_list)
from uncompyle6.scanners.tok import Token from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.consts import ( from uncompyle6.semantics.consts import (
@@ -453,39 +453,29 @@ class SourceWalker(GenericASTTraversal, object):
first = node[0][0] first = node[0][0]
else: else:
first = node[0] first = node[0]
pass
if first == 'tuple': if first == 'tuple':
flat_elems = []
# Note: don't iteratee over last element which is a # Note: don't iteratee over last element which is a
# BUILD_TUPLE... # BUILD_TUPLE...
for elem in first[:-1]: flat_elems = flatten_list(first[:-1], INDENT_PER_LEVEL)
if elem == 'expr1024':
for subelem in elem:
for subsubelem in subelem:
flat_elems.append(subsubelem)
elif elem == 'expr32':
for subelem in elem:
flat_elems.append(subelem)
else:
flat_elems.append(elem)
pass
pass
self.indent_more(INDENT_PER_LEVEL) self.indent_more(INDENT_PER_LEVEL)
sep = '' sep = ''
for elem in flat_elems: for elem in flat_elems:
if elem in ('ROT_THREE', 'EXTENDED_ARG'): if elem in ('ROT_THREE', 'EXTENDED_ARG'):
continue continue
assert elem == 'expr' assert elem == 'expr'
line_number = self.line_number line_number = self.line_number
value = self.traverse(elem) value = self.traverse(elem)
if line_number != self.line_number: if line_number != self.line_number:
sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1] sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
self.write(sep, value) self.write(sep, value)
sep = ', ' sep = ', '
self.indent_less(INDENT_PER_LEVEL) self.indent_less(INDENT_PER_LEVEL)
btuwc = node[-1] btuwc = node[-1]
assert btuwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL') assert btuwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
for n in node[1:-1]: for n in node[1:-1]:
@@ -1920,21 +1910,10 @@ class SourceWalker(GenericASTTraversal, object):
else: else:
raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack') raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')
flat_elems = [] flat_elems = flatten_list(node)
for elem in node:
if elem == 'expr1024':
for subelem in elem:
for subsubelem in subelem:
flat_elems.append(subsubelem)
elif elem == 'expr32':
for subelem in elem:
flat_elems.append(subelem)
else:
flat_elems.append(elem)
self.indent_more(INDENT_PER_LEVEL) self.indent_more(INDENT_PER_LEVEL)
sep = '' sep = ''
for elem in flat_elems: for elem in flat_elems:
if elem in ('ROT_THREE', 'EXTENDED_ARG'): if elem in ('ROT_THREE', 'EXTENDED_ARG'):
continue continue
@@ -1958,6 +1937,7 @@ class SourceWalker(GenericASTTraversal, object):
self.write(',') self.write(',')
self.write(endchar) self.write(endchar)
self.indent_less(INDENT_PER_LEVEL) self.indent_less(INDENT_PER_LEVEL)
self.prec = p self.prec = p
self.prune() self.prune()
return return