Merge pull request #165 from rocky/grammar-cleanup2

Grammar reduction for 2.6/2.7,3.x

pytest/test_grammar.py

@@ -18,21 +18,20 @@ def test_grammar():
      right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['expr1024', 'pos_arg', 'get_iter', 'attribute'])
+    expect_lhs = set(['pos_arg', 'get_iter', 'attribute'])
 
-    unused_rhs = set(['list', 'mkfunc',
+    unused_rhs = set(['list', 'mkfunc', 'dict',
                       'mklambda',
                       'unpack',])
     expect_right_recursive = set([('designList',
                                    ('store', 'DUP_TOP', 'designList'))])
+    expect_lhs.add('kvlist')
+    expect_lhs.add('kv3')
     if PYTHON3:
         expect_lhs.add('load_genexpr')
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
 
         unused_rhs = unused_rhs.union(set("""
         except_pop_except generator_exp
-        dict
         """.split()))
         if PYTHON_VERSION >= 3.0:
             expect_lhs.add("annotate_arg")

test/Makefile

@@ -139,7 +139,7 @@ grammar-coverage-2.6:
 grammar-coverage-2.7:
 	-rm $(COVER_DIR)/spark-grammar-2.7.cover || true
 	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
-	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=400
+	SPARK_PARSER_COVERAGE=$(COVER_DIR)/spark-grammar-2.7.cover $(PYTHON) test_pyenvlib.py --2.7.14 --max=600
 
 #: Get grammar coverage for Python 3.0
 grammar-coverage-3.0:

test/grammar-cover/grammar.sh

@@ -21,6 +21,7 @@ while [[ -n $1 ]]  ; do
     fi
 
     tmpdir=$workdir/../../tmp/grammar-cover
+    COVER_FILE=${tmpdir}/spark-grammar-${SHORT_VERSION}.cover
     [[ -d  $tmpdir ]] || mkdir $tmpdir
     cd $workdir/../..
     if [[ $SHORT_VERSION > 2.5 ]] ; then
@@ -31,10 +32,13 @@ while [[ -n $1 ]]  ; do
     GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
     pyenv local ${LONG_VERSION}
     cd ./test
+    if [[ -r $COVER_FILE ]]; then
+	rm $COVER_FILE
+    fi
     if [[ -r $GRAMMAR_TXT ]]; then
         GRAMMAR_SAVE_TXT=${tmpdir}/grammar-${SHORT_VERSION}-save.txt
         cp $GRAMMAR_TXT $GRAMMAR_SAVE_TXT
     fi
     make grammar-coverage-${SHORT_VERSION};
-    spark-parser-coverage --path ${tmpdir}/spark-grammar-${SHORT_VERSION}.cover > $GRAMMAR_TXT
+    spark-parser-coverage --max-count=3000 --path $COVER_FILE > $GRAMMAR_TXT
 done

uncompyle6/parser.py

@@ -567,9 +567,6 @@ class PythonParser(GenericASTBuilder):
 
         # Positional arguments in make_function
         pos_arg ::= expr
-
-        expr32 ::= expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr expr
-        expr1024 ::= expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32 expr32
         '''
 
     def p_store(self, args):

uncompyle6/parsers/parse2.py

@@ -109,11 +109,8 @@ class Python2Parser(PythonParser):
         _mklambda ::= load_closure mklambda
         kwarg     ::= LOAD_CONST expr
 
-        kvlist ::= kvlist kv3
         kv3 ::= expr expr STORE_MAP
 
-        dict ::= BUILD_MAP kvlist
-
         classdef ::= buildclass store
 
         buildclass ::= LOAD_CONST expr mkfunc

uncompyle6/parsers/parse25.py

@@ -65,7 +65,6 @@ class Python25Parser(Python26Parser):
         classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
         classdefdeco2 ::= LOAD_CONST expr mkfunc CALL_FUNCTION_0 BUILD_CLASS
         kv3 ::= expr expr STORE_MAP
-        kvlist ::= kvlist kv3
         ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
         return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP
         return_if_stmt   ::= ret_expr RETURN_END_IF POP_TOP

uncompyle6/parsers/parse26.py

@@ -261,6 +261,9 @@ class Python26Parser(Python2Parser):
 
     def p_misc26(self, args):
         """
+        dict ::= BUILD_MAP kvlist
+        kvlist ::= kvlist kv3
+
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and          ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
 

uncompyle6/parsers/parse27.py

@@ -38,8 +38,6 @@ class Python27Parser(Python2Parser):
         comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
         comp_iter     ::= comp_if
-        comp_iter     ::= comp_if_not
-        comp_if_not   ::= expr jmp_true comp_iter
         comp_iter     ::= comp_body
 
         dict_comp_body ::= expr expr MAP_ADD

uncompyle6/parsers/parse3.py

@@ -620,9 +620,7 @@ class Python3Parser(PythonParser):
                 self.addRule(rule, nop_func)
             elif opname.startswith('BUILD_LIST_UNPACK'):
                 v = token.attr
-                rule = ('build_list_unpack ::= ' + 'expr1024 ' * int(v//1024) +
+                rule = 'build_list_unpack ::= %s%s' % ('expr ' * v, opname)
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
                 self.addRule(rule, nop_func)
                 rule = 'expr ::= build_list_unpack'
                 self.addRule(rule, nop_func)
@@ -665,9 +663,7 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
                 v = token.attr
-                rule = ('build_map_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
+                rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
                 self.addRule(rule, nop_func)
             elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
                 v = token.attr
@@ -690,9 +686,7 @@ class Python3Parser(PythonParser):
                         self.add_unique_rule(rule, opname, token.attr, customize)
                 if not is_LOAD_CLOSURE or v == 0:
                     collection = opname_base[opname_base.find('_')+1:].lower()
-                    rule = (('%s ::= ' % collection) + 'expr1024 ' * int(v//1024) +
+                    rule = '%s ::= %s%s' % (collection, 'expr ' * v, opname)
-                            'expr32 ' * int((v//32) % 32) +
-                            'expr ' * (v % 32) + opname)
                     self.add_unique_rules([
                         'expr ::= %s' % collection,
                         rule], customize)