From b84c35acf572cfc0c460a6a0ab8e14fae34fabce Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 11 Dec 2017 21:26:58 -0500 Subject: [PATCH] DRY code; localize 3.{5,6} grammar rules.. helper.py, pysource: has code to flatten list used in n_list and n_build_tuple_unpack_with_call parse3{5,6}.py: localize grammar rules with BEFORE_ASYNC_WITH --- pytest/test_grammar.py | 3 +- uncompyle6/parsers/parse35.py | 44 ++++++++++++----------- uncompyle6/parsers/parse36.py | 61 ++++++++++++++++---------------- uncompyle6/semantics/helper.py | 22 ++++++++++++ uncompyle6/semantics/pysource.py | 56 ++++++++++------------------- 5 files changed, 96 insertions(+), 90 deletions(-) diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index 9fd0fa28..487deac7 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -35,6 +35,7 @@ def test_grammar(): expect_lhs.add("annotate_arg") expect_lhs.add("annotate_tuple") unused_rhs.add("mkfunc_annotate") + unused_rhs.add('call') if PYTHON_VERSION < 3.6: # 3.6 has at least one non-custom call rule # the others don't @@ -47,8 +48,6 @@ def test_grammar(): else: expect_right_recursive.add((('l_stmts', ('lastl_stmt', 'COME_FROM', 'l_stmts')))) - unused_rhs.add('build_map_unpack_with_call') - unused_rhs.add('unmapexpr') # expect_lhs.add('kwargs1') pass pass diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 5f78c016..78d1fcdb 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -57,24 +57,6 @@ class Python35Parser(Python34Parser): # Python 3.5+ async additions - stmt ::= async_with_stmt - async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts_opt - POP_BLOCK LOAD_CONST - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY - - stmt ::= async_with_as_stmt - async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store suite_stmts_opt - POP_BLOCK LOAD_CONST - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY - stmt ::= async_for_stmt async_for_stmt ::= SETUP_LOOP expr GET_AITER @@ -153,14 +135,36 @@ class Python35Parser(Python34Parser): nargs = token.attr % 256 map_unpack_n = "map_unpack_%s" % nargs rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs) - print("XXXX", rule) self.addRule(rule, nop_func) rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname) self.addRule(rule, nop_func) call_token = tokens[i+1] rule = 'call ::= expr unmapexpr ' + call_token.kind - print("XXXX2", rule) self.addRule(rule, nop_func) + elif opname == 'BEFORE_ASYNC_WITH': + rules_str = """ + async_with_stmt ::= expr + stmt ::= async_with_stmt + + async_with_stmt ::= expr + BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM + SETUP_ASYNC_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST + WITH_CLEANUP_START + GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + + stmt ::= async_with_as_stmt + + async_with_as_stmt ::= expr + BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM + SETUP_ASYNC_WITH store suite_stmts_opt + POP_BLOCK LOAD_CONST + WITH_CLEANUP_START + GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + """ + self.addRule(rules_str, nop_func) elif opname == 'BUILD_MAP_UNPACK': self.addRule(""" expr ::= unmap_dict diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index b351a802..0e33b561 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -39,23 +39,6 @@ class Python36Parser(Python35Parser): # Adds a COME_FROM_ASYNC_WITH over 3.5 # FIXME: remove corresponding rule for 3.5? - async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store - suite_stmts_opt - POP_BLOCK LOAD_CONST - COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY - async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts_opt - POP_BLOCK LOAD_CONST - COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM @@ -83,6 +66,30 @@ class Python36Parser(Python35Parser): fstring_single ::= expr FORMAT_VALUE """ self.add_unique_doc_rules(rules_str, customize) + elif opname == 'BEFORE_ASYNC_WITH': + rules_str = """ + stmt ::= async_with_stmt + async_with_as_stmt ::= expr + BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM + SETUP_ASYNC_WITH store + suite_stmts_opt + POP_BLOCK LOAD_CONST + COME_FROM_ASYNC_WITH + WITH_CLEANUP_START + GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + stmt ::= async_with_as_stmt + async_with_stmt ::= expr + BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM + SETUP_ASYNC_WITH POP_TOP suite_stmts_opt + POP_BLOCK LOAD_CONST + COME_FROM_ASYNC_WITH + WITH_CLEANUP_START + GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + """ + self.addRule(rules_str, nop_func) + elif opname == 'BUILD_STRING': v = token.attr joined_str_n = "formatted_value_%s" % v @@ -117,18 +124,6 @@ class Python36Parser(Python35Parser): 'expr32 ' * int((v//32) % 32) + 'expr ' * (v % 32) + opname) self.addRule(rule, nop_func) - elif opname.startswith('CALL_FUNCTION_EX'): - self.addRule("expr ::= call_ex", nop_func) - if token.attr == 0: - self.addRule(""" - unpack_list ::= list - call_ex ::= expr unpack_list CALL_FUNCTION_EX - """, nop_func) - else: - self.addRule( - 'call_ex ::= expr build_tuple_unpack_with_call CALL_FUNCTION_EX', - nop_func) - pass elif opname == 'CALL_FUNCTION_EX_KW': args_pos, args_kw = self.get_pos_kw(token) uniq_param = args_kw + args_pos @@ -137,6 +132,13 @@ class Python36Parser(Python35Parser): 'expr build_tuple_unpack_with_call build_map_unpack_with_call ' 'CALL_FUNCTION_EX_KW') self.addRule(rule, nop_func) + elif opname == 'CALL_FUNCTION_EX': + self.addRule("expr ::= call_ex", nop_func) + self.addRule(""" + unpack_list ::= list + call_ex ::= expr unpack_list CALL_FUNCTION_EX + """, nop_func) + pass else: super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, @@ -144,7 +146,6 @@ class Python36Parser(Python35Parser): seen_GET_AWAITABLE_YIELD_FROM, next_token) - class Python36ParserSingle(Python36Parser, PythonParserSingle): pass diff --git a/uncompyle6/semantics/helper.py b/uncompyle6/semantics/helper.py index 0ca81acc..a39bed8f 100644 --- a/uncompyle6/semantics/helper.py +++ b/uncompyle6/semantics/helper.py @@ -127,6 +127,28 @@ def print_docstring(self, indent, docstring): self.println(indent, trimmed[-1], quote) return True + +def flatten_list(node): + """ + List of expressions may be nested in groups of 32 and 1024 + items. flatten that out and return the list + """ + flat_elems = [] + for elem in node: + if elem == 'expr1024': + for subelem in elem: + for subsubelem in subelem: + flat_elems.append(subsubelem) + elif elem == 'expr32': + for subelem in elem: + flat_elems.append(subelem) + else: + flat_elems.append(elem) + pass + pass + return flat_elems + + # if __name__ == '__main__': # if PYTHON3: # from io import StringIO diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index e082d1cd..3d67af1a 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -127,7 +127,7 @@ from uncompyle6.semantics.make_function import ( from uncompyle6.semantics.parser_error import ParserError from uncompyle6.semantics.check_ast import checker from uncompyle6.semantics.helper import ( - print_docstring, find_globals) + print_docstring, find_globals, flatten_list) from uncompyle6.scanners.tok import Token from uncompyle6.semantics.consts import ( @@ -453,39 +453,29 @@ class SourceWalker(GenericASTTraversal, object): first = node[0][0] else: first = node[0] + pass if first == 'tuple': - flat_elems = [] # Note: don't iteratee over last element which is a # BUILD_TUPLE... - for elem in first[:-1]: - if elem == 'expr1024': - for subelem in elem: - for subsubelem in subelem: - flat_elems.append(subsubelem) - elif elem == 'expr32': - for subelem in elem: - flat_elems.append(subelem) - else: - flat_elems.append(elem) - pass - pass + flat_elems = flatten_list(first[:-1], INDENT_PER_LEVEL) - self.indent_more(INDENT_PER_LEVEL) - sep = '' + self.indent_more(INDENT_PER_LEVEL) + sep = '' - for elem in flat_elems: - if elem in ('ROT_THREE', 'EXTENDED_ARG'): - continue - assert elem == 'expr' - line_number = self.line_number - value = self.traverse(elem) - if line_number != self.line_number: - sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1] - self.write(sep, value) - sep = ', ' + for elem in flat_elems: + if elem in ('ROT_THREE', 'EXTENDED_ARG'): + continue + assert elem == 'expr' + line_number = self.line_number + value = self.traverse(elem) + if line_number != self.line_number: + sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1] + self.write(sep, value) + sep = ', ' self.indent_less(INDENT_PER_LEVEL) + btuwc = node[-1] assert btuwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL') for n in node[1:-1]: @@ -1920,21 +1910,10 @@ class SourceWalker(GenericASTTraversal, object): else: raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack') - flat_elems = [] - for elem in node: - if elem == 'expr1024': - for subelem in elem: - for subsubelem in subelem: - flat_elems.append(subsubelem) - elif elem == 'expr32': - for subelem in elem: - flat_elems.append(subelem) - else: - flat_elems.append(elem) + flat_elems = flatten_list(node) self.indent_more(INDENT_PER_LEVEL) sep = '' - for elem in flat_elems: if elem in ('ROT_THREE', 'EXTENDED_ARG'): continue @@ -1958,6 +1937,7 @@ class SourceWalker(GenericASTTraversal, object): self.write(',') self.write(endchar) self.indent_less(INDENT_PER_LEVEL) + self.prec = p self.prune() return