From daa424cf0c1726306de7dfc655200882ef5fb338 Mon Sep 17 00:00:00 2001
From: rocky <rb@dustyfeet.com>
Date: Sun, 17 Nov 2019 01:01:52 -0500
Subject: [PATCH] Some grammar cleanup based on coverage info

---
 uncompyle6/parsers/parse30.py | 17 ++++++++++++++---
 uncompyle6/parsers/parse31.py | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py
index df932d8a..cb042665 100644
--- a/uncompyle6/parsers/parse30.py
+++ b/uncompyle6/parsers/parse30.py
@@ -24,9 +24,7 @@ class Python30Parser(Python31Parser):
 
         # FIXME: combine with parse3.2
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt
-                              JUMP_BACK COME_FROM_LOOP
-        whileTruestmt     ::= SETUP_LOOP l_stmts_opt
-                              CONTINUE COME_FROM_LOOP
+                              jb_or_c COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP returns
                               COME_FROM_LOOP
 
@@ -205,6 +203,18 @@ class Python30Parser(Python31Parser):
 
     def remove_rules_30(self):
         self.remove_rules("""
+
+        # The were found using grammar coverage
+        while1stmt     ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+        whileTruestmt  ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
+        whileelsestmt  ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP
+        withasstmt     ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
+        withstmt       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
+        ##########################################################################################
+
         iflaststmtl        ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
         ifelsestmtl        ::= testexpr c_stmts_opt JUMP_BACK else_suitel
         iflaststmt         ::= testexpr c_stmts_opt JUMP_ABSOLUTE
@@ -224,6 +234,7 @@ class Python30Parser(Python31Parser):
         whileelsestmt      ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK
                                else_suitel COME_FROM_LOOP
 
+        ################################################################
         # No JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP,
         # POP_JUMP_IF_FALSE, or POP_JUMP_IF_TRUE
 
diff --git a/uncompyle6/parsers/parse31.py b/uncompyle6/parsers/parse31.py
index f36d7eb3..5e5b681c 100644
--- a/uncompyle6/parsers/parse31.py
+++ b/uncompyle6/parsers/parse31.py
@@ -32,11 +32,45 @@ class Python31Parser(Python32Parser):
         load  ::= LOAD_FAST
         load  ::= LOAD_NAME
         """
+    def remove_rules_31(self):
+        self.remove_rules("""
+        # DUP_TOP_TWO is DUP_TOPX in 3.1 and earlier
+        subscript2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
+        """)
 
     def customize_grammar_rules(self, tokens, customize):
         super(Python31Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules_31()
         return
     pass
 
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python31Parser()
+    p.remove_rules_31()
+    p.check_grammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.1:
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        ## FIXME: try this
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub(r'_\d+$', '',  t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))