More PyPy grammar rules

* assert one and two-arg form
* trystmt

Simplify adding multiple grammar rules

test/simple_source/bug_pypy27/01_assert2.py

@@ -0,0 +1,17 @@
+# From PyPy argparse, dedent.
+
+# PyPY adds opcode JUMP_IF_NOT_DEBUG.
+# This is the two argument form.
+assert __name__ != '__main"', 'Indent decreased below 0.'
+
+# From PyPy simple_interact.py
+# PyPy uses POP_JUMP_IF_FALSE as well as POP_JUMP_IF_TRUE
+# CPython only uses POP_JUMP_IF_TRUE
+while 1:
+    try:
+        more = 10
+    except EOFError:
+        break
+    more = len(__file__)
+    assert not more, "FOO"
+    assert not more

test/simple_source/bug_pypy27/03_try_return.py

@@ -0,0 +1,9 @@
+# From PyPy 2.7 argparse.py
+# PyPY reduces branches as a result of the return statement
+# So we need a new rules for trystmt and try_middle which we
+# suffix with _pypy, e.g. trystmt_pypy, and try_middle_pypy
+def call(self, string):
+    try:
+        return open(string, self, self._bufsize)
+    except IOError:
+        pass

uncompyle6/parser.py

@@ -30,6 +30,8 @@ class PythonParser(GenericASTBuilder):
 
     def add_unique_rule(self, rule, opname, count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
+           opname and count are used in the customize() semantic the actions
+           to add the semantic action rule. Often, count is not used.
         """
         if rule not in self.new_rules:
             # print("XXX ", rule) # debug
@@ -39,6 +41,14 @@ class PythonParser(GenericASTBuilder):
             pass
         return
 
+    def add_unique_rules(self, rules, customize):
+        """Add rules to grammar
+        """
+        for rule in rules:
+            opname = rule.split('::=')[0].strip()
+            self.add_unique_rule(rule, opname, 0, customize)
+        return
+
     def cleanup(self):
         """
         Remove recursive references to allow garbage

uncompyle6/parsers/parse2.py

@@ -247,9 +247,7 @@ class Python2Parser(PythonParser):
             expr ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP
             expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
 
-        For PYPY:
+        PyPy adds custom rules here as well
-            load_attr ::= expr LOOKUP_METHOD
-            call_function ::= expr CALL_METHOD
         '''
         for opname, v in list(customize.items()):
             opname_base = opname[:opname.rfind('_')]
@@ -272,37 +270,44 @@ class Python2Parser(PythonParser):
                                      opname, v, customize)
                 continue
             elif opname == 'JUMP_IF_NOT_DEBUG':
-                self.add_unique_rule(
+                self.add_unique_rules([
-                    "stmt ::= assert_pypy", opname_base, v, customize)
+                    'jmp_true_false ::= POP_JUMP_IF_TRUE',
-                self.add_unique_rule(
+                    'jmp_true_false ::= POP_JUMP_IF_FALSE',
-                    "stmt ::= assert2_pypy", opname_base, v, customize)
+                    "stmt ::= assert_pypy",
-                self.add_unique_rule(
+                    "stmt ::= assert2_pypy",
-                    "assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true "
+                    "assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false "
-                    "LOAD_ASSERT RAISE_VARARGS_1 COME_FROM",
+                       "LOAD_ASSERT RAISE_VARARGS_1 COME_FROM",
-                    opname_base, v, customize)
+                    "assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false "
-                self.add_unique_rule(
+                       "LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM",
-                    "assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true "
+                    ], customize)
-                    "LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM",
-                    opname_base, v, customize)
                 continue
             elif opname_base == 'BUILD_MAP':
                 if opname == 'BUILD_MAP_n':
                     # PyPy sometimes has no count. Sigh.
-                    rule = ('dictcomp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER designator '
+                    self.add_unique_rules([
-                            'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST')
+                        'dictcomp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER designator '
-                    self.add_unique_rule(rule, 'dictomp_func', 1, customize)
+                            'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST',
-
+                        'kvlist_n ::=  kvlist_n kv3',
-                    kvlist_n = 'kvlist_n'
+                        'kvlist_n ::=',
-                    rule = 'kvlist_n ::=  kvlist_n kv3'
+                        'mapexpr ::= BUILD_MAP_n kvlist_n',
-                    self.add_unique_rule(rule, 'kvlist_n', 0, customize)
+                    ], customize)
-                    rule = 'kvlist_n ::='
-                    self.add_unique_rule(rule, 'kvlist_n', 1, customize)
                 else:
                     kvlist_n = "kvlist_%s" % v
-                    rule = kvlist_n + ' ::= ' + ' kv3' * v
+                    self.add_unique_rules([
-                    self.add_unique_rule(rule, opname_base, v, customize)
+                        (kvlist_n + " ::=" + ' kv3' * v),
-                rule = "mapexpr ::=  %s %s" % (opname, kvlist_n)
+                        "mapexpr ::= %s %s" % (opname, kvlist_n)
-                self.add_unique_rule(rule, opname_base, v, customize)
+                    ], customize)
+                continue
+            elif opname == 'SETUP_EXCEPT':
+                # FIXME: have a way here to detect PyPy. Right now we
+                # only have SETUP_EXCEPT customization for PyPy, but that might not
+                # always be the case.
+                self.add_unique_rules([
+                    "stmt ::= trystmt_pypy",
+                    "trystmt_pypy ::= SETUP_EXCEPT suite_stmts_opt try_middle_pypy",
+                    "try_middle_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM"
+                    ], customize)
+                continue
             elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                 rule = 'unpack ::= ' + opname + ' designator'*v
             elif opname_base == 'UNPACK_LIST':
@@ -316,6 +321,7 @@ class Python2Parser(PythonParser):
                       ('pos_arg '*v, opname), nop_func)
                 rule = 'mkfunc ::= %s LOAD_CONST %s' % ('expr '*v, opname)
             elif opname_base == 'MAKE_CLOSURE':
+                # FIXME: use add_uniqe_rules to tidy this up.
                 self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' %
                       ('expr '*v, opname), nop_func)
                 self.addRule('genexpr ::= %s load_closure LOAD_GENEXPR %s expr GET_ITER CALL_FUNCTION_1' %

uncompyle6/parsers/parse27.py

@@ -66,7 +66,6 @@ class Python27Parser(Python2Parser):
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
         """
 
-
 class Python27ParserSingle(Python27Parser, PythonParserSingle):
     pass
 

uncompyle6/scanners/scanner2.py

@@ -97,10 +97,21 @@ class Scanner2(scan.Scanner):
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
         for i in self.op_range(0, n):
-            # We need to detect the difference between
+            # We need to detect the difference between:
-            # "raise AssertionError" and
+            #   raise AssertionError
-            # "assert"
+            #  and
-            if self.code[i] == self.opc.PJIT and self.code[i+3] == self.opc.LOAD_GLOBAL:
+            #   assert ...
+            # Below we use the heuristic that it is preceded by a POP_JUMP.
+            # however we could also use followed by RAISE_VARARGS
+            # or for PyPy there may be a JUMP_IF_NOT_DEBUG before.
+            # FIXME: remove uses of PJIF, and PJIT
+            if self.is_pypy:
+                have_pop_jump = self.code[i] in (self.opc.PJIF,
+                                                 self.opc.PJIT)
+            else:
+                have_pop_jump = self.code[i] == self.opc.PJIT
+
+            if have_pop_jump and self.code[i+3] == self.opc.LOAD_GLOBAL:
                 if names[self.get_argument(i+3)] == 'AssertionError':
                     self.load_asserts.add(i+3)
 
@@ -195,10 +206,12 @@ class Scanner2(scan.Scanner):
                         opname = '%s_%d' % (opname, oparg)
                     if op != self.opc.BUILD_SLICE:
                         customize[opname] = oparg
-            elif self.is_pypy and opname in ('LOOKUP_METHOD', 'JUMP_IF_NOT_DEBUG'):
+            elif self.is_pypy and opname in ('LOOKUP_METHOD',
-                # The value in the dict is used in rule uniquness key.
+                                             'JUMP_IF_NOT_DEBUG',
-                # These ops need only be done once. Hence we use arbitrary constant
+                                             'SETUP_EXCEPT'):
-                # 0.
+                # The value in the dict is in special cases in semantic actions, such
+                # as CALL_FUNCTION. The value is not used in these cases, so we put
+                # in arbitrary value 0.
                 customize[opname] = 0
             elif op == self.opc.JUMP_ABSOLUTE:
                 target = self.get_target(offset)

uncompyle6/scanners/scanner3.py

@@ -243,7 +243,10 @@ class Scanner3(scan.Scanner):
                 else:
                     opname = '%s_%d' % (opname, pos_args)
             elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'):
-                customize['CALL_METHOD'] = argval
+                # The value in the dict is in special cases in semantic actions, such
+                # as CALL_FUNCTION. The value is not used in these cases, so we put
+                # in arbitrary value 0.
+                customize[opname] = 0
             elif opname == 'UNPACK_EX':
                 # FIXME: try with scanner and parser by
                 # changing inst.argval

uncompyle6/semantics/pysource.py

@@ -385,7 +385,8 @@ TABLE_DIRECT = {
     # PyPy Additions
     #######################
     'assert_pypy':	( '%|assert %c\n' , 1 ),
-    'assert2_pypy':	( '%|assert %c, %c\n' , 1, 4 )
+    'assert2_pypy':	( '%|assert %c, %c\n' , 1, 4 ),
+    'trystmt_pypy':	( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
 }
 
 
@@ -1743,7 +1744,7 @@ class SourceWalker(GenericASTTraversal, object):
                 # Is there some sort of invalid bounds access going on?
                 if isinstance(entry[arg], int):
                     self.preorder(node[entry[arg]])
-                arg += 1
+                    arg += 1
             elif typ == 'p':
                 p = self.prec
                 (index, self.prec) = entry[arg]