From 7d8c17cb9358efbeb4bc879aeade195f080112b2 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 7 Dec 2017 12:27:22 -0500 Subject: [PATCH] grammar isolation and reduction --- pytest/test_grammar.py | 15 +++++++++++---- uncompyle6/parser.py | 6 ++---- uncompyle6/parsers/parse2.py | 2 ++ uncompyle6/parsers/parse3.py | 13 +++++++++---- uncompyle6/parsers/parse35.py | 4 ++-- uncompyle6/parsers/parse36.py | 4 ++++ 6 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index 138a5d33..9fd0fa28 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -20,8 +20,8 @@ def test_grammar(): unused_rhs = set(['list', 'mkfunc', 'mklambda', 'unpack',]) - expect_right_recursive = frozenset([('designList', - ('store', 'DUP_TOP', 'designList'))]) + expect_right_recursive = set([('designList', + ('store', 'DUP_TOP', 'designList'))]) if PYTHON3: expect_lhs.add('load_genexpr') expect_lhs.add('kvlist') @@ -35,14 +35,21 @@ def test_grammar(): expect_lhs.add("annotate_arg") expect_lhs.add("annotate_tuple") unused_rhs.add("mkfunc_annotate") - if PYTHON_VERSION != 3.6: + if PYTHON_VERSION < 3.6: # 3.6 has at least one non-custom call rule # the others don't unused_rhs.add('call') + if PYTHON_VERSION == 3.5: + expect_right_recursive.add((('l_stmts', + ('lastl_stmt', 'COME_FROM', 'l_stmts')))) + pass + pass else: - # These are custom rules set in 3.5 + expect_right_recursive.add((('l_stmts', + ('lastl_stmt', 'COME_FROM', 'l_stmts')))) unused_rhs.add('build_map_unpack_with_call') unused_rhs.add('unmapexpr') + # expect_lhs.add('kwargs1') pass pass pass diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index a578be12..bfcf7ea8 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -42,6 +42,7 @@ class PythonParser(GenericASTBuilder): 'kvlist_n', # Python 3.6+ 'joined_str', + 'come_from_loops', ] self.collect = frozenset(nt_list) @@ -157,9 +158,6 @@ class PythonParser(GenericASTBuilder): else: raise ParserError(None, -1) - def typestring(self, token): - return token.kind - def nonterminal(self, nt, args): n = len(args) @@ -177,6 +175,7 @@ class PythonParser(GenericASTBuilder): rv.append(args[1]) elif n == 1 and args[0] in self.singleton: rv = GenericASTBuilder.nonterminal(self, nt, args[0]) + del args[0] # save memory else: rv = GenericASTBuilder.nonterminal(self, nt, args) return rv @@ -225,7 +224,6 @@ class PythonParser(GenericASTBuilder): lastc_stmt ::= iflaststmt lastc_stmt ::= forelselaststmt lastc_stmt ::= ifelsestmtc - lastc_stmt ::= tryelsestmtc c_stmts_opt ::= c_stmts c_stmts_opt ::= passstmt diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 7f57b95a..e3381e98 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -133,6 +133,8 @@ class Python2Parser(PythonParser): POP_BLOCK LOAD_CONST COME_FROM suite_stmts_opt END_FINALLY + lastc_stmt ::= tryelsestmtc + # Move to 2.7? 2.6 may use come_froms tryelsestmtc ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK except_handler else_suitec COME_FROM diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 908e2e81..7776a409 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -104,6 +104,7 @@ class Python3Parser(PythonParser): kwarg ::= LOAD_CONST expr kwargs ::= kwarg* + kwargs1 ::= kwarg+ classdef ::= build_class store @@ -268,7 +269,7 @@ class Python3Parser(PythonParser): except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM_EXCEPT_CLAUSE - for_block ::= l_stmts_opt come_from_loops JUMP_BACK + for_block ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK for_block ::= l_stmts iflaststmtl ::= testexpr c_stmts_opt """ @@ -300,7 +301,6 @@ class Python3Parser(PythonParser): opt_come_from_except ::= come_from_except_clauses come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE+ - come_from_loops ::= COME_FROM_LOOP* """ def p_jump3(self, args): @@ -335,7 +335,7 @@ class Python3Parser(PythonParser): def p_loop_stmt3(self, args): """ forstmt ::= SETUP_LOOP expr _for store for_block POP_BLOCK - come_from_loops + COME_FROM_LOOP forelsestmt ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suite COME_FROM_LOOP @@ -812,7 +812,7 @@ class Python3Parser(PythonParser): opname, token.attr, customize) if args_kw > 0: - kwargs_str = 'kwargs ' + kwargs_str = 'kwargs1 ' else: kwargs_str = '' @@ -907,6 +907,11 @@ class Python3Parser(PythonParser): rule = ('mkfunc ::= kwargs %s%s %s' % ('pos_arg ' * args_pos, 'LOAD_CONST '*2, opname)) + elif self.version > 3.5: + # positional args before keyword args + rule = ('mkfunc ::= %skwargs1 %s %s' % + ('pos_arg ' * args_pos, 'LOAD_CONST '*2, + opname)) elif self.version > 3.3: # positional args before keyword args rule = ('mkfunc ::= %skwargs %s %s' % diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 06682934..9a57afe5 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -85,7 +85,7 @@ class Python35Parser(Python34Parser): POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK JUMP_ABSOLUTE END_FINALLY COME_FROM for_block POP_BLOCK JUMP_ABSOLUTE - come_from_loops + COME_FROM_LOOP async_for_stmt ::= SETUP_LOOP expr GET_AITER @@ -97,7 +97,7 @@ class Python35Parser(Python34Parser): POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK JUMP_ABSOLUTE END_FINALLY JUMP_BACK passstmt POP_BLOCK JUMP_ABSOLUTE - come_from_loops + COME_FROM_LOOP stmt ::= async_forelse_stmt async_forelse_stmt ::= SETUP_LOOP expr diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 9976e9a8..daaa4c05 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -37,6 +37,10 @@ class Python36Parser(Python35Parser): call ::= expr expr CALL_FUNCTION_EX call ::= expr expr expr CALL_FUNCTION_EX_KW_1 + for_block ::= l_stmts_opt come_from_loops JUMP_BACK + come_from_loops ::= COME_FROM_LOOP* + + # This might be valid in < 3.6 and ::= expr jmp_false expr