Handle PyPy CALL_METHOD op more correctly

Start testing pypy2.7 and 3.2 bytecodes

__pkginfo__.py

@@ -37,7 +37,7 @@ entry_points={
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.4.0',
-                      'xdis >= 2.0.1']
+                      'xdis >= 2.0.2']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

requirements.txt

@@ -1,2 +1,2 @@
 spark-parser >= 1.2.1
-xdis >= 2.0.1
+xdis >= 2.0.2

test/Makefile

@@ -112,7 +112,15 @@ check-3.4-ok:
 
 #: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
 # Skip for now
-2.4 2.6 5.0:
+2.6:
+
+#: PyPy 5.0.x with ...
+5.0:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy2.7 --verify
+
+#: PyPy 2.4.x with ...
+2.4:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
 clean: clean-py-dis clean-dis clean-unverified
 

test/simple_source/bug27+/05_setattr.py

@@ -0,0 +1,9 @@
+# Ensure PyPy handling of:
+#    key, value in slotstate.items().
+# PyPy uses LOOKUP_METHOD and CALL_METHOD instead
+# of LOAD_ATTR and CALL_FUNCTION
+def bug(state, slotstate):
+    if state:
+        if slotstate is not None:
+            for key, value in slotstate.items():
+                setattr(state, key, 2)

uncompyle6/parsers/parse2.py

@@ -266,14 +266,12 @@ class Python2Parser(PythonParser):
                     self.seen1024 = True
                 rule = ('build_list ::= ' + 'expr1024 '*thousands +
                                         'expr32 '*thirty32s + 'expr '*(v%32) + opname)
-            elif opname == 'CALL_METHOD':
+            elif opname == 'LOOKUP_METHOD':
-                # A PyPy speciality
+                # A PyPy speciality - DRY with parse3
                 self.add_unique_rule("load_attr ::= LOAD_FAST LOOKUP_METHOD",
-                                     opname_base, v, customize)
+                                     opname, v, customize)
                 self.add_unique_rule("load_attr ::= LOAD_NAME LOOKUP_METHOD",
-                                     opname_base, v, customize)
+                                     opname, v, customize)
-                self.add_unique_rule("call_function ::= expr CALL_METHOD",
-                                     opname_base, v, customize)
                 continue
             elif opname == 'JUMP_IF_NOT_DEBUG':
                 self.add_unique_rule(
@@ -314,11 +312,19 @@ class Python2Parser(PythonParser):
                 # rule = 'mkfunc ::= %s closure_list LOAD_CONST %s' % ('expr '*v, opname)
             elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
                                  'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
-                na = (v & 0xff)           # positional parameters
+                args_pos = (v & 0xff)          # positional parameters
-                nk = (v >> 8) & 0xff      # keyword parameters
+                args_kw = (v >> 8) & 0xff      # keyword parameters
                 # number of apply equiv arguments:
                 nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3
-                rule = 'call_function ::= expr ' + 'expr '*na + 'kwarg '*nk \
+                rule = 'call_function ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \
+                       + 'expr ' * nak + opname
+            elif opname_base == 'CALL_METHOD':
+                # PyPy only - DRY with parse3
+                args_pos = (v & 0xff)           # positional parameters
+                args_kw = (v >> 8) & 0xff      # keyword parameters
+                # number of apply equiv arguments:
+                nak = ( len(opname_base)-len('CALL_METHOD') ) // 3
+                rule = 'call_function ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \
                        + 'expr ' * nak + opname
             else:
                 raise Exception('unknown customize token %s' % opname)

uncompyle6/parsers/parse3.py

@@ -462,14 +462,12 @@ class Python3Parser(PythonParser):
                 if opname_base == 'BUILD_TUPLE':
                     rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
-            elif opname == 'CALL_METHOD':
+            elif opname == 'LOOKUP_METHOD':
-                # A PyPy speciality
+                # A PyPy speciality - DRY with parse2
                 self.add_unique_rule("load_attr ::= LOAD_FAST LOOKUP_METHOD",
                                      opname, token.attr, customize)
                 self.add_unique_rule("load_attr ::= LOAD_NAME LOOKUP_METHOD",
                                      opname, token.attr, customize)
-                self.add_unique_rule("call_function ::= expr CALL_METHOD",
-                                     opname, token.attr, customize)
                 continue
             elif opname == 'JUMP_IF_NOT_DEBUG':
                 self.add_unique_rule(
@@ -526,6 +524,17 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= kwargs %sexpr %s' %
                             ('pos_arg ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname_base == 'CALL_METHOD':
+                # PyPy only - DRY with parse2
+                args_pos = (token.attr & 0xff)          # positional parameters
+                args_kw = (token.attr >> 8) & 0xff      # keyword parameters
+                # number of apply equiv arguments:
+                nak = ( len(opname_base)-len('CALL_METHOD') ) // 3
+                rule = ('call_function ::= expr '
+                        + ('pos_arg ' * args_pos)
+                        + ('kwarg ' * args_kw)
+                        + 'expr ' * nak + token.type)
+                self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname.startswith('MAKE_CLOSURE'):
                 # DRY with MAKE_FUNCTION
                 # Note: this probably doesn't handle kwargs proprerly

uncompyle6/scanners/scanner2.py

@@ -192,8 +192,11 @@ class Scanner2(scan.Scanner):
                     opname = '%s_%d' % (opname, oparg)
                     if op != self.opc.BUILD_SLICE:
                         customize[opname] = oparg
-            elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'):
+            elif self.is_pypy and opname in ('LOOKUP_METHOD', 'JUMP_IF_NOT_DEBUG'):
-                customize[opname] = oparg
+                # The value in the dict is used in rule uniquness key.
+                # These ops need only be done once. Hence we use arbitrary constant
+                # 0.
+                customize[opname] = 0
             elif op == self.opc.JUMP_ABSOLUTE:
                 target = self.get_target(offset)
                 if target < offset:

uncompyle6/scanners/scanner27.py

@@ -52,7 +52,7 @@ class Scanner27(Scanner2):
         # opcodes with expect a variable number pushed values whose
         # count is in the opcode. For parsing we generally change the
         # opcode name to include that number.
-        self.varargs_ops = frozenset([
+        varargs_ops = set([
             self.opc.BUILD_LIST,           self.opc.BUILD_TUPLE,
             self.opc.BUILD_SLICE,          self.opc.UNPACK_SEQUENCE,
             self.opc.MAKE_FUNCTION,        self.opc.CALL_FUNCTION,
@@ -62,6 +62,10 @@ class Scanner27(Scanner2):
             # New in Python 2.7
             self.opc.BUILD_SET,            self.opc.BUILD_MAP])
 
+        if is_pypy:
+            varargs_ops.add(self.opc.CALL_METHOD)
+        self.varargs_ops = frozenset(varargs_ops)
+
         # "setup" opcodes
         self.setup_ops = frozenset([
             self.opc.SETUP_EXCEPT, self.opc.SETUP_FINALLY,

uncompyle6/scanners/scanner3.py

@@ -89,14 +89,16 @@ class Scanner3(scan.Scanner):
 
         # Opcodes that take a variable number of arguments
         # (expr's)
-        self.varargs = frozenset([self.opc.BUILD_LIST,
+        varargs_ops = set([
-                                  self.opc.BUILD_TUPLE,
+            self.opc.BUILD_LIST,        self.opc.BUILD_TUPLE,
-                                  self.opc.BUILD_SET,
+             self.opc.BUILD_SET,        self.opc.BUILD_SLICE,
-                                  self.opc.BUILD_SLICE,
+             self.opc.BUILD_MAP,        self.opc.UNPACK_SEQUENCE,
-                                  self.opc.BUILD_MAP,
-                                  self.opc.UNPACK_SEQUENCE,
              self.opc.RAISE_VARARGS])
 
+        if is_pypy:
+            varargs_ops.add(self.opc.CALL_METHOD)
+        self.varargs_ops = frozenset(varargs_ops)
+
         # Not really a set, but still clasification-like
         self.statement_opcode_sequences = [
             (self.opc.POP_JUMP_IF_FALSE, self.opc.JUMP_FORWARD),
@@ -234,7 +236,7 @@ class Scanner3(scan.Scanner):
                     )
                 )
                 continue
-            elif op in self.varargs:
+            elif op in self.varargs_ops:
                 pos_args = inst.argval
                 opname = '%s_%d' % (opname, pos_args)
             elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'):