PyPy support

* Use proper PYPY 32 opcodes
* handle opcodes LOOKUP_METHOD and CALL_METHOD
* Administrative stuff for PyPy

Makefile

@@ -40,6 +40,10 @@ check-3.2 check-3.5:
 check-2.6:
 	$(MAKE) -C test $@
 
+#:PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
+# Skip for now
+2.6 5.0:
+
 #: Run py.test tests
 pytest:
 	$(MAKE) -C pytest check

__pkginfo__.py

@@ -37,7 +37,7 @@ entry_points={
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.4.0',
-                      'xdis >= 2.0.0']
+                      'xdis >= 2.0.1']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

pytest/test_fjt.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from uncompyle6 import PYTHON_VERSION
+from uncompyle6 import PYTHON_VERSION, IS_PYPY
 from uncompyle6.scanner import get_scanner
 from array import array
 def bug(state, slotstate):
@@ -12,7 +12,7 @@ def test_if_in_for():
     code = bug.__code__
     scan = get_scanner(PYTHON_VERSION)
     print(PYTHON_VERSION)
-    if 2.7 <= PYTHON_VERSION <= 3.0:
+    if 2.7 <= PYTHON_VERSION <= 3.0 and not IS_PYPY:
         n = scan.setup_code(code)
         scan.build_lines_data(code, n)
         scan.build_prev_op(n)

requirements.txt

@@ -1,2 +1,2 @@
 spark-parser >= 1.2.1
-xdis >= 2.0.0
+xdis >= 2.0.1

test/Makefile

@@ -110,6 +110,10 @@ check-3.2-ok:
 check-3.4-ok:
 	$(PYTHON) test_pythonlib.py --ok-3.4 --verify $(COMPILE)
 
+#:PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4]
+# Skip for now
+2.6 5.0:
+
 clean: clean-py-dis clean-dis clean-unverified
 
 clean-dis:

test/test_pythonlib.py

@@ -78,13 +78,13 @@ for  vers in (2.7, 3.4, 3.5):
     test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers)
     pass
 
-for  vers in (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5):
+for  vers in (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 'pypy3.2', 'pypy2.7'):
     bytecode = "bytecode_%s" % vers
     key = "bytecode-%s" % vers
     test_options[key] =  (bytecode, PYC, bytecode, vers)
     key = "%s" % vers
     pythonlib = "python%s" % vers
-    if vers >= 3.0:
+    if isinstance(vers, float) and vers >= 3.0:
         pythonlib = os.path.join(pythonlib, '__pycache__')
     test_options[key] =  (os.path.join(lib_prefix, pythonlib), PYOC, pythonlib, vers)
 

uncompyle6/parsers/parse2.py

@@ -230,7 +230,7 @@ class Python2Parser(PythonParser):
 
     def add_custom_rules(self, tokens, customize):
         '''
-        Special handling for opcodes that take a variable number
+        Special handling for opcodes such as those that take a variable number
         of arguments -- we add a new rule for each:
 
             build_list  ::= {expr}^n BUILD_LIST_n
@@ -246,6 +246,10 @@ class Python2Parser(PythonParser):
             expr ::= expr {expr}^n CALL_FUNCTION_VAR_n POP_TOP
             expr ::= expr {expr}^n CALL_FUNCTION_VAR_KW_n POP_TOP
             expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
+
+        For PYPY:
+            load_attr ::= LOAD_FAST LOOKUP_METHOD
+            call_function ::= expr CALL_METHOD
         '''
         for k, v in list(customize.items()):
             op = k[:k.rfind('_')]
@@ -262,6 +266,13 @@ class Python2Parser(PythonParser):
                     self.seen1024 = True
                 rule = ('build_list ::= ' + 'expr1024 '*thousands +
                                         'expr32 '*thirty32s + 'expr '*(v%32) + k)
+            elif k == 'CALL_METHOD':
+                # A PyPy speciality
+                self.add_unique_rule("load_attr ::= LOAD_FAST LOOKUP_METHOD",
+                                     op, v, customize)
+                self.add_unique_rule("call_function ::= expr CALL_METHOD",
+                                     op, v, customize)
+                continue
             elif op == 'BUILD_MAP':
                 kvlist_n = "kvlist_%s" % v
                 rule = kvlist_n + ' ::= ' + ' kv3' * v

uncompyle6/parsers/parse3.py

@@ -385,7 +385,7 @@ class Python3Parser(PythonParser):
 
     def add_custom_rules(self, tokens, customize):
         """
-        Special handling for opcodes that take a variable number
+        Special handling for opcodes such as those that take a variable number
         of arguments -- we add a new rule for each:
 
             unpack_list ::= UNPACK_LIST_n {expr}^n
@@ -436,7 +436,10 @@ class Python3Parser(PythonParser):
             mkfunc   ::= {pos_arg}^n [LOAD_CONST] MAKE_FUNCTION_n
             mklambda ::= {pos_arg}^n LOAD_LAMBDA [LOAD_CONST] MAKE_FUNCTION_n
 
-        """
+        For PYPY:
+            load_attr ::= LOAD_FAST LOOKUP_METHOD
+            call_function ::= expr CALL_METHOD
+       """
         for i, token in enumerate(tokens):
             opname = token.type
             opname_base = opname[:opname.rfind('_')]
@@ -448,19 +451,6 @@ class Python3Parser(PythonParser):
                 rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
                             "GET_ITER CALL_FUNCTION_1")
                 self.add_make_function_rule(rule_pat, opname, token.attr, customize)
-            ## Custom rules which are handled now by the more generic rule in
-            ## either MAKE_FUNCTION or MAKE_CLOSURE
-            # elif opname == 'LOAD_GENEXPR':
-            #     rule_pat = ("genexpr ::= LOAD_GENEXPR %sMAKE_FUNCTION_0 expr "
-            #                 "GET_ITER CALL_FUNCTION_1")
-            #     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
-            #     rule_pat = ("genexpr ::= load_closure LOAD_GENEXPR %sMAKE_CLOSURE_0 expr "
-            #                 "GET_ITER CALL_FUNCTION_1")
-            #     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
-            # elif opname == 'LOAD_LISTCOMP':
-            #     rule_pat  = ("listcomp ::= LOAD_LISTCOMP %sMAKE_FUNCTION_0 expr "
-            #                 "GET_ITER CALL_FUNCTION_1")
-            #     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
             elif opname == 'LOAD_SETCOMP':
                 # Should this be generalized and put under MAKE_FUNCTION?
                 rule_pat = ("setcomp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr "
@@ -476,6 +466,13 @@ class Python3Parser(PythonParser):
                 if opname_base == 'BUILD_TUPLE':
                     rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
+            elif opname == 'CALL_METHOD':
+                # A PyPy speciality
+                self.add_unique_rule("load_attr ::= LOAD_FAST LOOKUP_METHOD",
+                                     opname, token.attr, customize)
+                self.add_unique_rule("call_function ::= expr CALL_METHOD",
+                                     opname, token.attr, customize)
+                continue
             elif opname_base == 'BUILD_MAP':
                 kvlist_n = "kvlist_%s" % token.attr
                 if self.version >= 3.5:

uncompyle6/scanner.py

@@ -51,9 +51,10 @@ class Scanner(object):
     def __init__(self, version, show_asm=None, is_pypy=False):
         self.version = version
         self.show_asm = show_asm
+        self.is_pypy = is_pypy
 
         if version in PYTHON_VERSIONS:
-            if is_pypy and version != 3.2:
+            if is_pypy:
                 v_str = "opcode_pypy%s" % (int(version * 10))
             else:
                 v_str = "opcode_%s" % (int(version * 10))

uncompyle6/scanners/scanner2.py

@@ -41,14 +41,18 @@ class Scanner2(scan.Scanner):
 
     def disassemble(self, co, classname=None, code_objects={}, show_asm=None):
         """
-        Disassemble a Python 2 code object, returning a list of 'Token'.
+        Pick out tokens from an uncompyle6 code object, and transform them,
-        Various tranformations are made to assist the deparsing grammar.
+        returning a list of uncompyle6 'Token's.
-        For example:
+
+        The tranformations are made to assist the deparsing grammar.
+        Specificially:
            -  various types of LOAD_CONST's are categorized in terms of what they load
            -  COME_FROM instructions are added to assist parsing control structures
-           -  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional aruments
+           -  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
-        The main part of this procedure is modelled after
+
-        dis.disassemble().
+        Also, when we encounter certain tokens, we add them to a set which will cause custom
+        grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
+        cause specific rules for the specific number of arguments they take.
         """
 
         show_asm = self.show_asm if not show_asm else show_asm
@@ -188,6 +192,8 @@ class Scanner2(scan.Scanner):
                     opname = '%s_%d' % (opname, oparg)
                     if op != self.opc.BUILD_SLICE:
                         customize[opname] = oparg
+            elif self.is_pypy and opname == 'CALL_METHOD':
+                customize[opname] = oparg
             elif op == self.opc.JUMP_ABSOLUTE:
                 target = self.get_target(offset)
                 if target < offset:

uncompyle6/scanners/scanner3.py

@@ -107,14 +107,18 @@ class Scanner3(scan.Scanner):
 
     def disassemble(self, co, classname=None, code_objects={}, show_asm=None):
         """
-        Disassemble a Python 3 code object, returning a list of 'Token'.
+        Pick out tokens from an uncompyle6 code object, and transform them,
-        Various tranformations are made to assist the deparsing grammar.
+        returning a list of uncompyle6 'Token's.
-        For example:
+
+        The tranformations are made to assist the deparsing grammar.
+        Specificially:
            -  various types of LOAD_CONST's are categorized in terms of what they load
            -  COME_FROM instructions are added to assist parsing control structures
-           -  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional aruments
+           -  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
-        The main part of this procedure is modelled after
+
-        dis.disassemble().
+        Also, when we encounter certain tokens, we add them to a set which will cause custom
+        grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
+        cause specific rules for the specific number of arguments they take.
         """
 
         show_asm = self.show_asm if not show_asm else show_asm

uncompyle6/semantics/pysource.py

@@ -1802,13 +1802,18 @@ class SourceWalker(GenericASTTraversal, object):
 
     def customize(self, customize):
         """
-        Special handling for opcodes that take a variable number
+        Special handling for opcodes, such as those that take a variable number
         of arguments -- we add a new entry for each in TABLE_R.
         """
         for k, v in list(customize.items()):
             if k in TABLE_R:
                 continue
             op = k[ :k.rfind('_') ]
+
+            if k == 'CALL_METHOD':
+                # This happens in PyPy only
+                TABLE_R[k] = ('%c(%P)', 0, (1, -1, ', ', 100))
+
             if op == 'CALL_FUNCTION':
                 TABLE_R[k] = ('%c(%P)', 0, (1, -1, ', ', 100))
             elif op in ('CALL_FUNCTION_VAR',