Administrivia

* Some grammar cleanup
* add more grammar debugging

.travis.yml

@@ -5,6 +5,7 @@ python:
   - '2.7'
   - '3.4'
   - '3.6'
+  - '3.8'
 
 matrix:
   include:

Makefile

@@ -47,8 +47,12 @@ check-3.8:
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:
 
-#:PyPy pypy3-2.4.0 Python 3:
-pypy-3.2 2.4:
+#:PyPy pypy3-2.4.0 Python 3.6.1:
+7.1 pypy-3.2 2.4:
+	$(MAKE) -C test $@
+
+#:PyPy pypy3-2.4.0 Python 3.6.9:
+7.2:
 	$(MAKE) -C test $@
 
 #: Run py.test tests

NEWS.md

@@ -1,4 +1,16 @@
-4.1.0 2019-10-12 Stony Brook Ride
+3.5.1 2019-10-29 JNC
+====================
+
+- Pypy 3.3, 3.5, 3.6, and 3.6.9 support
+- Improve 3.0 decompilation
+    - no parse errors on stlib bytecode. However accurate translation in
+	  control-flow and and/or detection needs work
+- Remove extraneous iter() in "for" of list comprehension  Fixes #272
+- "for" block without a POP_BLOCK and confusing JUMP_BACK for CONTINUE. Fixes #293
+- Fix unmarshal incompletness detected in Pypy 3.6
+- Miscellaneous bugs fixed
+
+3.5.0 2019-10-12 Stony Brook Ride
 =================================
 
 - Fix fragment bugs

__pkginfo__.py

@@ -58,7 +58,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ["spark-parser >= 1.8.9, < 1.9.0",
-                      "xdis >= 4.1.0, < 4.2.0"]
+                      "xdis >= 4.1.3, < 4.2.0"]
 
 license            = "GPL3"
 mailing_list       = "python-debugger@googlegroups.com"

admin-tools/how-to-make-a-release.md

@@ -56,19 +56,21 @@
 
     $ . ./admin-tools/make-dist-older.sh
     $ git tag release-python-2.4-$VERSION
-
+	$ twine check dist/uncompyle6-$VERSION*
     $ . ./admin-tools/make-dist-newer.sh
-
-Goto https://github.com/rocky/python-uncompyle6/releases
+	$ twine check dist/uncompyle6-$VERSION*
 
 # Upload single package and look at Rst Formating
 
+	$ twine check dist/uncompyle6-${VERSION}*
     $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg
 
 # Upload rest of versions
 
     $ twine upload dist/uncompyle6-${VERSION}*
 
+Goto https://github.com/rocky/python-uncompyle6/releases
+
 # Push tags:
 
     $ git push --tags

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.9 3.7.4 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.10 3.5.7'
+export PYVERSIONS='3.5.9 3.6.9 2.6.9 3.3.7 2.7.17 3.2.6 3.1.5 3.4.10 3.7.5'

admin-tools/setup-master.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-PYTHON_VERSION=3.6.8
+PYTHON_VERSION=3.7.5
 
 # FIXME put some of the below in a common routine
 function finish {

pytest/test_grammar.py

@@ -1,50 +1,54 @@
 import re
-from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
+from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY  # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser, python_parser
 from uncompyle6.scanner import get_scanner
 
-def test_grammar():
 
+def test_grammar():
     def check_tokens(tokens, opcode_set):
         remain_tokens = set(tokens) - opcode_set
-        remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('LOAD_CODE$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
-        assert remain_tokens == set([]), \
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
+        assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % (
+            remain_tokens,
+            p.dump_grammar(),
+        )
 
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    (lhs, rhs, tokens,
-     right_recursive, dup_rhs) = p.check_sets()
+    (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets()
 
     # We have custom rules that create the below
-    expect_lhs = set(['pos_arg', 'attribute'])
+    expect_lhs = set(["pos_arg", "attribute"])
     if PYTHON_VERSION < 3.8:
-        expect_lhs.add('get_iter')
+        expect_lhs.add("get_iter")
+    else:
+        expect_lhs.add("async_with_as_stmt")
+        expect_lhs.add("async_with_stmt")
 
+    unused_rhs = set(["list", "mkfunc", "mklambda", "unpack"])
 
-    unused_rhs = set(['list', 'mkfunc',
-                      'mklambda',
-                      'unpack',])
-
-    expect_right_recursive = set([('designList',
-                                   ('store', 'DUP_TOP', 'designList'))])
+    expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))])
 
     if PYTHON_VERSION < 3.7:
-        unused_rhs.add('call')
+        unused_rhs.add("call")
 
     if PYTHON_VERSION > 2.6:
-        expect_lhs.add('kvlist')
-        expect_lhs.add('kv3')
-        unused_rhs.add('dict')
+        expect_lhs.add("kvlist")
+        expect_lhs.add("kv3")
+        unused_rhs.add("dict")
 
     if PYTHON3:
-        expect_lhs.add('load_genexpr')
+        expect_lhs.add("load_genexpr")
 
-        unused_rhs = unused_rhs.union(set("""
+        unused_rhs = unused_rhs.union(
+            set(
+                """
         except_pop_except generator_exp
-        """.split()))
+        """.split()
+            )
+        )
         if PYTHON_VERSION >= 3.0:
             expect_lhs.add("annotate_arg")
             expect_lhs.add("annotate_tuple")
@@ -53,17 +57,19 @@ def test_grammar():
             unused_rhs.add("classdefdeco1")
             unused_rhs.add("tryelsestmtl")
             if PYTHON_VERSION >= 3.5:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'come_froms', 'l_stmts'))))
+                expect_right_recursive.add(
+                    (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts")))
+                )
                 pass
             elif 3.0 < PYTHON_VERSION < 3.3:
-                expect_right_recursive.add((('l_stmts',
-                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                expect_right_recursive.add(
+                    (("l_stmts", ("lastl_stmt", "COME_FROM", "l_stmts")))
+                )
                 pass
             pass
         pass
     else:
-        expect_lhs.add('kwarg')
+        expect_lhs.add("kwarg")
 
     assert expect_lhs == set(lhs)
 
@@ -73,9 +79,16 @@ def test_grammar():
 
     assert expect_right_recursive == right_recursive
 
-    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
-                                ('LOAD_CONST',),
-                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    expect_dup_rhs = frozenset(
+        [
+            ("COME_FROM",),
+            ("CONTINUE",),
+            ("JUMP_ABSOLUTE",),
+            ("LOAD_CONST",),
+            ("JUMP_BACK",),
+            ("JUMP_FORWARD",),
+        ]
+    )
     reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs)
     for k in reduced_dup_rhs:
         print(k, reduced_dup_rhs[k])
@@ -83,7 +96,7 @@ def test_grammar():
 
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
     ignore_set = set(
-            """
+        """
             JUMP_BACK CONTINUE
             COME_FROM COME_FROM_EXCEPT
             COME_FROM_EXCEPT_CLAUSE
@@ -92,22 +105,33 @@ def test_grammar():
             LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
-            """.split())
+            """.split()
+    )
     if 2.6 <= PYTHON_VERSION <= 2.7:
         opcode_set = set(s.opc.opname).union(ignore_set)
         if PYTHON_VERSION == 2.6:
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION == 3.4:
-        ignore_set.add('LOAD_CLASSNAME')
-        ignore_set.add('STORE_LOCALS')
+        ignore_set.add("LOAD_CLASSNAME")
+        ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)
         check_tokens(tokens, opcode_set)
 
+
 def test_dup_rule():
     import inspect
-    python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
-                  is_pypy=IS_PYPY,
-                  parser_debug={
-                      'dups': True, 'transition': False, 'reduce': False,
-                      'rules': False, 'errorstack': None, 'context': True})
+
+    python_parser(
+        PYTHON_VERSION,
+        inspect.currentframe().f_code,
+        is_pypy=IS_PYPY,
+        parser_debug={
+            "dups": True,
+            "transition": False,
+            "reduce": False,
+            "rules": False,
+            "errorstack": None,
+            "context": True,
+        },
+    )

setup.py

@@ -32,6 +32,7 @@ setup(
        install_requires   = install_requires,
        license            = license,
        long_description   = long_description,
+       long_description_content_type = "text/x-rst",
        name               = modname,
        packages           = find_packages(),
        py_modules         = py_modules,

test/Makefile

@@ -99,8 +99,8 @@ check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
 	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
-        --bytecode-3.7 --bytecode-3.8 \
-	--bytecode-pypy3.2
+        --bytecode-3.7  \
+	--bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x
 check-bytecode-3-short:
@@ -332,8 +332,16 @@ pypy-2.7 5.0 5.3 6.0:
 pypy-3.2 2.4:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
 
-#: PyPy 5.0.x with Python 3.6 ...
+#: PyPy 5.0.x with Python 3.6.1 ...
+check-bytecode-pypy3.6: 7.1
 7.1:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
+
+#: PyPy 5.0.x with Python 3.6.9
+check-bytecode-pypy3.6: 7.2
+7.2:
+	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
 	$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
 
 

test/simple_source/bug30/02_try_except_except.py

@@ -7,3 +7,13 @@ def start_new_thread(function, args, kwargs={}):
         pass
     except:
         args()
+
+# Adapted from 3.0.1 code.py
+# Bug is again JUMP_FORWARD elimination compared
+# to earlier and later Pythons.
+def interact():
+    while 1:
+        try:
+            more = 1
+        except KeyboardInterrupt:
+            more = 0

test/simple_source/bug30/02_while1_if_while1.py

@@ -7,3 +7,8 @@ while 1:
             raise RuntimeError
     else:
         raise RuntimeError
+
+# Adapted from 3.0.1 cgi.py
+def _parseparam(s, end):
+    while end > 0 and s.count(''):
+        end = s.find(';')

test/simple_source/bug30/03_ifelse.py

@@ -0,0 +1,50 @@
+# Adapted from 3.0 base64
+# Problem was handling if/else which
+# needs to be like Python 2.6 (and not like 2.7 or 3.1)
+def main(args, f, func, sys):
+    """Small main program"""
+    if args and args[0] != '-':
+        func(f, sys.stdout.buffer)
+    else:
+        func(sys.stdin.buffer, sys.stdout.buffer)
+
+# From Python 3.0 _markupbase.py.
+#
+# The Problem was in the way "if"s are generated in 3.0 which is sort
+# of like a more optimized Python 2.6, with reduced extraneous jumps,
+# but still 2.6-ish and not 2.7- or 3.1-ish.
+def parse_marked_section(fn, i, rawdata, report=1):
+    if report:
+        j = 1
+        fn(rawdata[i: j])
+    return 10
+
+# From 3.0.1 _abcoll.py
+# Bug was in genexpr_func which doesn't have a JUMP_BACK but
+# in its gen_comp_body, we can use COME_FROM in its place.
+# As above omission of JUMPs is a feature of 3.0 that doesn't
+# seem to be in later versions (or earlier like 2.6).
+def __and__(self, other, Iterable):
+    if not isinstance(other, Iterable):
+        return NotImplemented
+    return self._from_iterable(value for value in other if value in self)
+
+# Adapted from 3.0.1 abc.py
+# Bug was in handling multiple COME_FROMs in return_if_stmt
+def __instancecheck__(subtype, subclass, cls):
+    if subtype:
+        if (cls and subclass):
+            return False
+
+
+# Adapted from 3.0.1 abc.py
+# Bug was rule in "jump_absolute_else" and disallowing
+# "else" to the wrong place.
+
+def _strptime(locale_time, found_zone, time):
+    for tz_values in locale_time:
+        if found_zone:
+            if (time and found_zone):
+                break
+            else:
+                break

test/simple_source/bug30/04_and_del.py

@@ -18,3 +18,13 @@ assert normpath(['.']) == []
 assert normpath(['a', 'b', '..']) == ['a']
 assert normpath(['a', 'b', '', 'c']) == ['a', 'b', 'c']
 assert normpath(['a', 'b', '.', '', 'c', '..']) == ['a', 'b']
+
+# Adapted from 3.0.1 cgi.py
+# Bug was in detecting "or" and "and" due to lack of PUSH/POP_IF instructions.
+def handle(format, html, text):
+    formatter = (format and html) or text
+    return formatter
+
+assert handle(False, False, True)
+assert not handle(True, False, False)
+assert handle(True, True, False)

test/simple_source/bug38/01_extra_iter.py

@@ -0,0 +1,7 @@
+# Adapted from From 3.3 urllib/parse.py
+qs = "https://travis-ci.org/rocky/python-uncompyle6/builds/605260823?utm_medium=notification&utm_source=email"
+expect = ['https://travis-ci.org/rocky/python-uncompyle6/builds/605260823?utm_medium=notification', 'utm_source=email']
+
+# Should visually see that we don't add an extra iter() which is not technically wrong, just
+# unnecessary.
+assert expect == [s2 for s1 in qs.split('&') for s2 in s1.split(';')]

test/simple_source/bug38/01_for_continue.py

@@ -0,0 +1,5 @@
+# Bug is turning a JUMP_BACK for a CONTINUE so for has no JUMP_BACK.
+# Also there is no POP_BLOCK since there isn't anything in the loop.
+# In the future when we have better control flow, we might redo all of this.
+for i in range(2):
+    pass

test/test_pyenvlib.py

@@ -43,6 +43,9 @@ TEST_VERSIONS = (
     "pypy3.5-5.7.1-beta",
     "pypy3.5-5.9.0",
     "pypy3.5-6.0.0",
+    "pypy3.6-7.1.0",
+    "pypy3.6-7.1.1",
+    "pypy3.6-7.2.0",
     "native",
 ) + tuple(python_versions)
 

uncompyle6/disas.py

@@ -34,12 +34,11 @@ from __future__ import print_function
 import sys
 from collections import deque
 
-import uncompyle6
-
 from xdis.code import iscode
 from xdis.load import check_object_path, load_module
 from uncompyle6.scanner import get_scanner
 
+
 def disco(version, co, out=None, is_pypy=False):
     """
     diassembles and deparses a given code block 'co'
@@ -49,10 +48,9 @@ def disco(version, co, out=None, is_pypy=False):
 
     # store final output stream for case of error
     real_out = out or sys.stdout
-    print('# Python %s' % version, file=real_out)
+    print("# Python %s" % version, file=real_out)
     if co.co_filename:
-        print('# Embedded file name: %s' % co.co_filename,
-              file=real_out)
+        print("# Embedded file name: %s" % co.co_filename, file=real_out)
 
     scanner = get_scanner(version, is_pypy=is_pypy)
 
@@ -63,10 +61,12 @@ def disco(version, co, out=None, is_pypy=False):
 def disco_loop(disasm, queue, real_out):
     while len(queue) > 0:
         co = queue.popleft()
-        if co.co_name != '<module>':
-            print('\n# %s line %d of %s' %
-                      (co.co_name, co.co_firstlineno, co.co_filename),
-                      file=real_out)
+        if co.co_name != "<module>":
+            print(
+                "\n# %s line %d of %s"
+                % (co.co_name, co.co_firstlineno, co.co_filename),
+                file=real_out,
+            )
         tokens, customize = disasm(co)
         for t in tokens:
             if iscode(t.pattr):
@@ -77,6 +77,7 @@ def disco_loop(disasm, queue, real_out):
             pass
         pass
 
+
 # def disassemble_fp(fp, outstream=None):
 #     """
 #     disassemble Python byte-code from an open file
@@ -90,6 +91,7 @@ def disco_loop(disasm, queue, real_out):
 #         disco(version, co, outstream, is_pypy=is_pypy)
 #     co = None
 
+
 def disassemble_file(filename, outstream=None):
     """
     disassemble Python byte-code file (.pyc)
@@ -98,8 +100,7 @@ def disassemble_file(filename, outstream=None):
     try to find the corresponding compiled object.
     """
     filename = check_object_path(filename)
-    (version, timestamp, magic_int, co, is_pypy,
-     source_size) = load_module(filename)
+    (version, timestamp, magic_int, co, is_pypy, source_size) = load_module(filename)
     if type(co) == list:
         for con in co:
             disco(version, con, outstream)
@@ -107,6 +108,7 @@ def disassemble_file(filename, outstream=None):
         disco(version, co, outstream, is_pypy=is_pypy)
     co = None
 
+
 def _test():
     """Simple test program to disassemble a file."""
     argc = len(sys.argv)

uncompyle6/parsers/parse2.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2018 Rocky Bernstein
+#  Copyright (c) 2015-2019 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
 #  Copyright (c) 1999 John Aycock
@@ -31,10 +31,10 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
-class Python2Parser(PythonParser):
 
+class Python2Parser(PythonParser):
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
-        super(Python2Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser)
+        super(Python2Parser, self).__init__(SyntaxTree, "stmts", debug=debug_parser)
         self.new_rules = set()
 
     def p_print2(self, args):
@@ -52,7 +52,7 @@ class Python2Parser(PythonParser):
         """
 
     def p_print_to(self, args):
-        '''
+        """
         stmt ::= print_to
         stmt ::= print_to_nl
         stmt ::= print_nl_to
@@ -62,10 +62,10 @@ class Python2Parser(PythonParser):
         print_to_items ::= print_to_items print_to_item
         print_to_items ::= print_to_item
         print_to_item ::= DUP_TOP expr ROT_TWO PRINT_ITEM_TO
-        '''
+        """
 
     def p_grammar(self, args):
-        '''
+        """
         sstmt ::= stmt
         sstmt ::= return RETURN_LAST
 
@@ -176,12 +176,12 @@ class Python2Parser(PythonParser):
         jmp_abs ::= JUMP_ABSOLUTE
         jmp_abs ::= JUMP_BACK
         jmp_abs ::= CONTINUE
-        '''
+        """
 
     def p_generator_exp2(self, args):
-        '''
+        """
         generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
-        '''
+        """
 
     def p_expr2(self, args):
         """
@@ -252,25 +252,41 @@ class Python2Parser(PythonParser):
         this.
         """
 
-        if 'PyPy' in customize:
+        if "PyPy" in customize:
             # PyPy-specific customizations
-            self.addRule("""
+            self.addRule(
+                """
                         stmt ::= assign3_pypy
                         stmt ::= assign2_pypy
                         assign3_pypy ::= expr expr expr store store store
                         assign2_pypy ::= expr expr store store
                         list_comp    ::= expr  BUILD_LIST_FROM_ARG for_iter store list_iter
                                          JUMP_BACK
-                        """, nop_func)
+                        """,
+                nop_func,
+            )
 
         # For a rough break out on the first word. This may
         # include instructions that don't need customization,
         # but we'll do a finer check after the rough breakout.
         customize_instruction_basenames = frozenset(
-            ('BUILD',     'CALL',       'CONTINUE',  'DELETE',
-             'DUP',       'EXEC',       'GET',       'JUMP',
-             'LOAD',      'LOOKUP',     'MAKE',      'SETUP',
-             'RAISE',     'UNPACK'))
+            (
+                "BUILD",
+                "CALL",
+                "CONTINUE",
+                "DELETE",
+                "DUP",
+                "EXEC",
+                "GET",
+                "JUMP",
+                "LOAD",
+                "LOOKUP",
+                "MAKE",
+                "SETUP",
+                "RAISE",
+                "UNPACK",
+            )
+        )
 
         # Opcode names in the custom_seen_ops set have rules that get added
         # unconditionally and the rules are constant. So they need to be done
@@ -284,139 +300,191 @@ class Python2Parser(PythonParser):
 
             # Do a quick breakout before testing potentially
             # each of the dozen or so instruction in if elif.
-            if (opname[:opname.find('_')] not in customize_instruction_basenames
-                    or opname in custom_seen_ops):
+            if (
+                opname[: opname.find("_")] not in customize_instruction_basenames
+                or opname in custom_seen_ops
+            ):
                 continue
 
-            opname_base = opname[:opname.rfind('_')]
+            opname_base = opname[: opname.rfind("_")]
 
             # The order of opname listed is roughly sorted below
-            if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
+            if opname_base in ("BUILD_LIST", "BUILD_SET", "BUILD_TUPLE"):
                 # We do this complicated test to speed up parsing of
                 # pathelogically long literals, especially those over 1024.
                 build_count = token.attr
-                thousands = (build_count//1024)
-                thirty32s = ((build_count//32) % 32)
+                thousands = build_count // 1024
+                thirty32s = (build_count // 32) % 32
                 if thirty32s > 0:
-                    rule = "expr32 ::=%s" % (' expr' * 32)
+                    rule = "expr32 ::=%s" % (" expr" * 32)
                     self.add_unique_rule(rule, opname_base, build_count, customize)
                 if thousands > 0:
-                    self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32),
-                                         opname_base, build_count, customize)
-                collection = opname_base[opname_base.find('_')+1:].lower()
-                rule = (('%s ::= ' % collection) + 'expr1024 '*thousands +
-                        'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname)
-                self.add_unique_rules([
-                    "expr ::= %s" % collection,
-                    rule], customize)
+                    self.add_unique_rule(
+                        "expr1024 ::=%s" % (" expr32" * 32),
+                        opname_base,
+                        build_count,
+                        customize,
+                    )
+                collection = opname_base[opname_base.find("_") + 1 :].lower()
+                rule = (
+                    ("%s ::= " % collection)
+                    + "expr1024 " * thousands
+                    + "expr32 " * thirty32s
+                    + "expr " * (build_count % 32)
+                    + opname
+                )
+                self.add_unique_rules(["expr ::= %s" % collection, rule], customize)
                 continue
-            elif opname_base == 'BUILD_MAP':
-                if opname == 'BUILD_MAP_n':
+            elif opname_base == "BUILD_MAP":
+                if opname == "BUILD_MAP_n":
                     # PyPy sometimes has no count. Sigh.
-                    self.add_unique_rules([
-                        'kvlist_n ::=  kvlist_n kv3',
-                        'kvlist_n ::=',
-                        'dict ::= BUILD_MAP_n kvlist_n',
-                    ], customize)
+                    self.add_unique_rules(
+                        [
+                            "kvlist_n ::=  kvlist_n kv3",
+                            "kvlist_n ::=",
+                            "dict ::= BUILD_MAP_n kvlist_n",
+                        ],
+                        customize,
+                    )
                     if self.version >= 2.7:
                         self.add_unique_rule(
-                            'dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store '
-                            'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST',
-                            'dict_comp_func', 0, customize)
+                            "dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store "
+                            "comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST",
+                            "dict_comp_func",
+                            0,
+                            customize,
+                        )
 
                 else:
-                    kvlist_n =  ' kv3' * token.attr
+                    kvlist_n = " kv3" * token.attr
                     rule = "dict ::= %s%s" % (opname, kvlist_n)
                     self.addRule(rule, nop_func)
                 continue
-            elif opname_base == 'BUILD_SLICE':
-                slice_num  = token.attr
+            elif opname_base == "BUILD_SLICE":
+                slice_num = token.attr
                 if slice_num == 2:
-                     self.add_unique_rules([
-                        'expr ::= build_slice2',
-                        'build_slice2 ::= expr expr BUILD_SLICE_2'
-                        ], customize)
+                    self.add_unique_rules(
+                        [
+                            "expr ::= build_slice2",
+                            "build_slice2 ::= expr expr BUILD_SLICE_2",
+                        ],
+                        customize,
+                    )
                 else:
-                    assert slice_num == 3, ("BUILD_SLICE value must be 2 or 3; is %s" %
-                                            slice_num)
-                    self.add_unique_rules([
-                        'expr ::= build_slice3',
-                        'build_slice3 ::= expr expr expr BUILD_SLICE_3',
-                        ], customize)
+                    assert slice_num == 3, (
+                        "BUILD_SLICE value must be 2 or 3; is %s" % slice_num
+                    )
+                    self.add_unique_rules(
+                        [
+                            "expr ::= build_slice3",
+                            "build_slice3 ::= expr expr expr BUILD_SLICE_3",
+                        ],
+                        customize,
+                    )
                 continue
-            elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
-                                 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
+            elif opname_base in (
+                "CALL_FUNCTION",
+                "CALL_FUNCTION_VAR",
+                "CALL_FUNCTION_VAR_KW",
+                "CALL_FUNCTION_KW",
+            ):
 
                 args_pos, args_kw = self.get_pos_kw(token)
 
                 # number of apply equiv arguments:
-                nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3
-                rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \
-                       + 'expr ' * nak + opname
-            elif opname_base == 'CALL_METHOD':
+                nak = (len(opname_base) - len("CALL_FUNCTION")) // 3
+                rule = (
+                    "call ::= expr "
+                    + "expr " * args_pos
+                    + "kwarg " * args_kw
+                    + "expr " * nak
+                    + opname
+                )
+            elif opname_base == "CALL_METHOD":
                 # PyPy only - DRY with parse3
 
                 args_pos, args_kw = self.get_pos_kw(token)
 
                 # number of apply equiv arguments:
-                nak = ( len(opname_base)-len('CALL_METHOD') ) // 3
-                rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \
-                       + 'expr ' * nak + opname
-            elif opname == 'CONTINUE_LOOP':
-                self.addRule('continue ::= CONTINUE_LOOP', nop_func)
+                nak = (len(opname_base) - len("CALL_METHOD")) // 3
+                rule = (
+                    "call ::= expr "
+                    + "expr " * args_pos
+                    + "kwarg " * args_kw
+                    + "expr " * nak
+                    + opname
+                )
+            elif opname == "CONTINUE_LOOP":
+                self.addRule("continue ::= CONTINUE_LOOP", nop_func)
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'DELETE_ATTR':
-                self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func)
+            elif opname == "DELETE_ATTR":
+                self.addRule("del_stmt ::= expr DELETE_ATTR", nop_func)
                 custom_seen_ops.add(opname)
                 continue
-            elif opname.startswith('DELETE_SLICE'):
-                self.addRule("""
+            elif opname.startswith("DELETE_SLICE"):
+                self.addRule(
+                    """
                 del_expr ::= expr
                 del_stmt ::= del_expr DELETE_SLICE+0
                 del_stmt ::= del_expr del_expr DELETE_SLICE+1
                 del_stmt ::= del_expr del_expr DELETE_SLICE+2
                 del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3
-                """, nop_func)
+                """,
+                    nop_func,
+                )
                 custom_seen_ops.add(opname)
-                self.check_reduce['del_expr'] = 'AST'
+                self.check_reduce["del_expr"] = "AST"
                 continue
-            elif opname == 'DELETE_DEREF':
-                self.addRule("""
+            elif opname == "DELETE_DEREF":
+                self.addRule(
+                    """
                    stmt           ::= del_deref_stmt
                    del_deref_stmt ::= DELETE_DEREF
-                   """, nop_func)
+                   """,
+                    nop_func,
+                )
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'DELETE_SUBSCR':
-                self.addRule("""
+            elif opname == "DELETE_SUBSCR":
+                self.addRule(
+                    """
                     del_stmt ::= delete_subscript
                     delete_subscript ::= expr expr DELETE_SUBSCR
-                   """, nop_func)
-                self.check_reduce['delete_subscript'] = 'AST'
+                   """,
+                    nop_func,
+                )
+                self.check_reduce["delete_subscript"] = "AST"
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'GET_ITER':
-                self.addRule("""
+            elif opname == "GET_ITER":
+                self.addRule(
+                    """
                     expr      ::= get_iter
                     attribute ::= expr GET_ITER
-                    """, nop_func)
+                    """,
+                    nop_func,
+                )
                 custom_seen_ops.add(opname)
                 continue
-            elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'):
+            elif opname_base in ("DUP_TOPX", "RAISE_VARARGS"):
                 # FIXME: remove these conditions if they are not needed.
                 # no longer need to add a rule
                 continue
-            elif opname == 'EXEC_STMT':
-                self.addRule("""
+            elif opname == "EXEC_STMT":
+                self.addRule(
+                    """
                     stmt      ::= exec_stmt
                     exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
                     exec_stmt ::= expr exprlist EXEC_STMT
                     exprlist  ::= expr+
-                    """, nop_func)
+                    """,
+                    nop_func,
+                )
                 continue
-            elif opname == 'JUMP_IF_NOT_DEBUG':
-                self.addRule("""
+            elif opname == "JUMP_IF_NOT_DEBUG":
+                self.addRule(
+                    """
                     jmp_true_false ::= POP_JUMP_IF_TRUE
                     jmp_true_false ::= POP_JUMP_IF_FALSE
                     stmt ::= assert_pypy
@@ -426,107 +494,152 @@ class Python2Parser(PythonParser):
                     assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false
                                      LOAD_ASSERT expr CALL_FUNCTION_1
                                      RAISE_VARARGS_1 COME_FROM
-                     """, nop_func)
+                     """,
+                    nop_func,
+                )
                 continue
-            elif opname == 'LOAD_ATTR':
-                self.addRule("""
+            elif opname == "LOAD_ATTR":
+                self.addRule(
+                    """
                   expr      ::= attribute
                   attribute ::= expr LOAD_ATTR
-                  """, nop_func)
+                  """,
+                    nop_func,
+                )
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'LOAD_LISTCOMP':
+            elif opname == "LOAD_LISTCOMP":
                 self.addRule("expr ::= listcomp", nop_func)
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'LOAD_SETCOMP':
-                self.add_unique_rules([
-                    "expr ::= set_comp",
-                    "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1"
-                    ], customize)
+            elif opname == "LOAD_SETCOMP":
+                self.add_unique_rules(
+                    [
+                        "expr ::= set_comp",
+                        "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1",
+                    ],
+                    customize,
+                )
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'LOOKUP_METHOD':
+            elif opname == "LOOKUP_METHOD":
                 # A PyPy speciality - DRY with parse3
-                self.addRule("""
+                self.addRule(
+                    """
                              expr      ::= attribute
                              attribute ::= expr LOOKUP_METHOD
                              """,
-                             nop_func)
+                    nop_func,
+                )
                 custom_seen_ops.add(opname)
                 continue
-            elif opname_base == 'MAKE_FUNCTION':
-                if i > 0 and tokens[i-1] == 'LOAD_LAMBDA':
-                    self.addRule('mklambda ::= %s LOAD_LAMBDA %s' %
-                                 ('pos_arg ' * token.attr, opname), nop_func)
-                rule = 'mkfunc ::= %s LOAD_CODE %s' % ('expr ' * token.attr, opname)
-            elif opname_base == 'MAKE_CLOSURE':
+            elif opname_base == "MAKE_FUNCTION":
+                if i > 0 and tokens[i - 1] == "LOAD_LAMBDA":
+                    self.addRule(
+                        "mklambda ::= %s LOAD_LAMBDA %s"
+                        % ("pos_arg " * token.attr, opname),
+                        nop_func,
+                    )
+                rule = "mkfunc ::= %s LOAD_CODE %s" % ("expr " * token.attr, opname)
+            elif opname_base == "MAKE_CLOSURE":
                 # FIXME: use add_unique_rules to tidy this up.
-                if i > 0 and tokens[i-1] == 'LOAD_LAMBDA':
-                    self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' %
-                                 ('expr ' * token.attr, opname),  nop_func)
+                if i > 0 and tokens[i - 1] == "LOAD_LAMBDA":
+                    self.addRule(
+                        "mklambda ::= %s load_closure LOAD_LAMBDA %s"
+                        % ("expr " * token.attr, opname),
+                        nop_func,
+                    )
                 if i > 0:
-                    prev_tok = tokens[i-1]
-                    if prev_tok == 'LOAD_GENEXPR':
-                        self.add_unique_rules([
-                            ('generator_exp ::= %s load_closure LOAD_GENEXPR %s expr'
-                                 ' GET_ITER CALL_FUNCTION_1' %
-                            ('expr ' * token.attr, opname))], customize)
+                    prev_tok = tokens[i - 1]
+                    if prev_tok == "LOAD_GENEXPR":
+                        self.add_unique_rules(
+                            [
+                                (
+                                    "generator_exp ::= %s load_closure LOAD_GENEXPR %s expr"
+                                    " GET_ITER CALL_FUNCTION_1"
+                                    % ("expr " * token.attr, opname)
+                                )
+                            ],
+                            customize,
+                        )
                         pass
-                self.add_unique_rules([
-                    ('mkfunc ::= %s load_closure LOAD_CODE %s' %
-                     ('expr ' * token.attr, opname))], customize)
+                self.add_unique_rules(
+                    [
+                        (
+                            "mkfunc ::= %s load_closure LOAD_CODE %s"
+                            % ("expr " * token.attr, opname)
+                        )
+                    ],
+                    customize,
+                )
 
                 if self.version >= 2.7:
                     if i > 0:
-                        prev_tok = tokens[i-1]
-                        if prev_tok == 'LOAD_DICTCOMP':
-                            self.add_unique_rules([
-                                ('dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr'
-                                 ' GET_ITER CALL_FUNCTION_1' %
-                                ('expr ' * token.attr, opname))], customize)
-                        elif prev_tok == 'LOAD_SETCOMP':
-                            self.add_unique_rules([
-                                "expr ::= set_comp",
-                                ('set_comp ::= %s load_closure LOAD_SETCOMP %s expr'
-                                ' GET_ITER CALL_FUNCTION_1' %
-                                ('expr ' * token.attr, opname))
-                                ], customize)
+                        prev_tok = tokens[i - 1]
+                        if prev_tok == "LOAD_DICTCOMP":
+                            self.add_unique_rules(
+                                [
+                                    (
+                                        "dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr"
+                                        " GET_ITER CALL_FUNCTION_1"
+                                        % ("expr " * token.attr, opname)
+                                    )
+                                ],
+                                customize,
+                            )
+                        elif prev_tok == "LOAD_SETCOMP":
+                            self.add_unique_rules(
+                                [
+                                    "expr ::= set_comp",
+                                    (
+                                        "set_comp ::= %s load_closure LOAD_SETCOMP %s expr"
+                                        " GET_ITER CALL_FUNCTION_1"
+                                        % ("expr " * token.attr, opname)
+                                    ),
+                                ],
+                                customize,
+                            )
                         pass
                     pass
                 continue
-            elif opname == 'SETUP_EXCEPT':
-                if 'PyPy' in customize:
-                    self.add_unique_rules([
-                        "stmt ::= try_except_pypy",
-                        "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy",
-                        "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM"
-                        ], customize)
+            elif opname == "SETUP_EXCEPT":
+                if "PyPy" in customize:
+                    self.add_unique_rules(
+                        [
+                            "stmt ::= try_except_pypy",
+                            "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy",
+                            "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM",
+                        ],
+                        customize,
+                    )
                 custom_seen_ops.add(opname)
                 continue
-            elif opname == 'SETUP_FINALLY':
-                if 'PyPy' in customize:
-                    self.addRule("""
+            elif opname == "SETUP_FINALLY":
+                if "PyPy" in customize:
+                    self.addRule(
+                        """
                         stmt ::= tryfinallystmt_pypy
                         tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY
-                                                suite_stmts_opt END_FINALLY""", nop_func)
+                                                suite_stmts_opt END_FINALLY""",
+                        nop_func,
+                    )
 
                 custom_seen_ops.add(opname)
                 continue
-            elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
+            elif opname_base in ("UNPACK_TUPLE", "UNPACK_SEQUENCE"):
                 custom_seen_ops.add(opname)
-                rule = 'unpack ::= ' + opname + ' store' * token.attr
-            elif opname_base == 'UNPACK_LIST':
+                rule = "unpack ::= " + opname + " store" * token.attr
+            elif opname_base == "UNPACK_LIST":
                 custom_seen_ops.add(opname)
-                rule = 'unpack_list ::= ' + opname + ' store' * token.attr
+                rule = "unpack_list ::= " + opname + " store" * token.attr
             else:
                 continue
             self.addRule(rule, nop_func)
             pass
 
-        self.check_reduce['raise_stmt1'] = 'tokens'
-        self.check_reduce['aug_assign2'] = 'AST'
-        self.check_reduce['or'] = 'AST'
+        self.check_reduce["raise_stmt1"] = "tokens"
+        self.check_reduce["aug_assign2"] = "AST"
+        self.check_reduce["or"] = "AST"
         # self.check_reduce['_stmts'] = 'AST'
 
         # Dead code testing...
@@ -541,24 +654,30 @@ class Python2Parser(PythonParser):
         # Dead code testing...
         # if lhs == 'while1elsestmt':
         #     from trepan.api import debug; debug()
-        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
+        if (
+            lhs in ("aug_assign1", "aug_assign2")
+            and ast[0]
+            and ast[0][0] in ("and", "or")
+        ):
             return True
-        elif lhs in ('raise_stmt1',):
+        elif lhs in ("raise_stmt1",):
             # We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule
-            return (tokens[first] == 'LOAD_ASSERT' and (last >= len(tokens)))
-        elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
+            return tokens[first] == "LOAD_ASSERT" and (last >= len(tokens))
+        elif rule == ("or", ("expr", "jmp_true", "expr", "\\e_come_from_opt")):
             expr2 = ast[2]
-            return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
-        elif lhs in ('delete_subscript', 'del_expr'):
+            return expr2 == "expr" and expr2[0] == "LOAD_ASSERT"
+        elif lhs in ("delete_subscript", "del_expr"):
             op = ast[0][0]
-            return op.kind in ('and', 'or')
+            return op.kind in ("and", "or")
 
         return False
 
+
 class Python2ParserSingle(Python2Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Check grammar
     p = Python2Parser()
     p.check_grammar()

uncompyle6/parsers/parse26.py

@@ -254,7 +254,7 @@ class Python26Parser(Python2Parser):
                          POP_TOP jb_pb_come_from
 
         generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
-        list_if ::= list_if ::= expr jmp_false_then list_iter
+        list_if ::= expr jmp_false_then list_iter
         '''
 
     def p_ret26(self, args):
@@ -467,7 +467,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 2.6:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse27.py

@@ -41,8 +41,6 @@ class Python27Parser(Python2Parser):
         comp_body ::= set_comp_body
         comp_for ::= expr for_iter store comp_iter JUMP_BACK
 
-        comp_iter     ::= comp_body
-
         dict_comp_body ::= expr expr MAP_ADD
         set_comp_body ::= expr SET_ADD
 
@@ -294,7 +292,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 2.7:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse3.py

@@ -86,10 +86,8 @@ class Python3Parser(PythonParser):
         dict_comp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER store
                            comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
 
-        comp_iter     ::= comp_if
         comp_iter     ::= comp_if_not
         comp_if_not   ::= expr jmp_true comp_iter
-        comp_iter     ::= comp_body
         """
 
     def p_grammar(self, args):
@@ -432,7 +430,7 @@ class Python3Parser(PythonParser):
         else:
             return "%s_0" % (token.kind)
 
-    def custom_build_class_rule(self, opname, i, token, tokens, customize):
+    def custom_build_class_rule(self, opname, i, token, tokens, customize, is_pypy):
         """
         # Should the first rule be somehow folded into the 2nd one?
         build_class ::= LOAD_BUILD_CLASS mkfunc
@@ -483,10 +481,18 @@ class Python3Parser(PythonParser):
             call_function = call_fn_tok.kind
             if call_function.startswith("CALL_FUNCTION_KW"):
                 self.addRule("classdef ::= build_class_kw store", nop_func)
-                rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" % (
-                    "expr " * (call_fn_tok.attr - 1),
-                    call_function,
-                )
+                if is_pypy:
+                    args_pos, args_kw = self.get_pos_kw(call_fn_tok)
+                    rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %s%s%s" % (
+                        "expr " * (args_pos - 1),
+                        "kwarg " * (args_kw),
+                        call_function,
+                    )
+                else:
+                    rule = (
+                        "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s"
+                        % ("expr " * (call_fn_tok.attr - 1), call_function)
+                    )
             else:
                 call_function = self.call_fn_name(call_fn_tok)
                 rule = "build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" % (
@@ -496,7 +502,7 @@ class Python3Parser(PythonParser):
         self.addRule(rule, nop_func)
         return
 
-    def custom_classfunc_rule(self, opname, token, customize, next_token):
+    def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
         """
         call ::= expr {expr}^n CALL_FUNCTION_n
         call ::= expr {expr}^n CALL_FUNCTION_VAR_n
@@ -514,18 +520,28 @@ class Python3Parser(PythonParser):
         # Yes, this computation based on instruction name is a little bit hoaky.
         nak = (len(opname) - len("CALL_FUNCTION")) // 3
 
-        token.kind = self.call_fn_name(token)
         uniq_param = args_kw + args_pos
 
         # Note: 3.5+ have subclassed this method; so we don't handle
         # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here.
-        rule = (
-            "call ::= expr "
-            + ("pos_arg " * args_pos)
-            + ("kwarg " * args_kw)
-            + "expr " * nak
-            + token.kind
-        )
+        if is_pypy and self.version >= 3.6:
+            if token == "CALL_FUNCTION":
+                token.kind = self.call_fn_name(token)
+            rule = (
+                "call ::= expr "
+                + ("pos_arg " * args_pos)
+                + ("kwarg " * args_kw)
+                + token.kind
+            )
+        else:
+            token.kind = self.call_fn_name(token)
+            rule = (
+                "call ::= expr "
+                + ("pos_arg " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+            )
 
         self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
@@ -543,7 +559,12 @@ class Python3Parser(PythonParser):
         this has an effect on many rules.
         """
         if self.version >= 3.3:
-            new_rule = rule % (("LOAD_STR ") * 1)
+            if PYTHON3 or not self.is_pypy:
+                load_op = "LOAD_STR "
+            else:
+                load_op = "LOAD_CONST "
+
+            new_rule = rule % ((load_op) * 1)
         else:
             new_rule = rule % (("LOAD_STR ") * 0)
         self.add_unique_rule(new_rule, opname, attr, customize)
@@ -571,7 +592,7 @@ class Python3Parser(PythonParser):
 
         """
 
-        is_pypy = False
+        self.is_pypy = False
 
         # For a rough break out on the first word. This may
         # include instructions that don't need customization,
@@ -616,7 +637,7 @@ class Python3Parser(PythonParser):
         # a specific instruction seen.
 
         if "PyPy" in customize:
-            is_pypy = True
+            self.is_pypy = True
             self.addRule(
                 """
               stmt ::= assign3_pypy
@@ -821,7 +842,9 @@ class Python3Parser(PythonParser):
                         """
                     self.addRule(rule, nop_func)
 
-                self.custom_classfunc_rule(opname, token, customize, tokens[i + 1])
+                self.custom_classfunc_rule(
+                    opname, token, customize, tokens[i + 1], self.is_pypy
+                )
                 # Note: don't add to custom_ops_processed.
 
             elif opname_base == "CALL_METHOD":
@@ -880,21 +903,30 @@ class Python3Parser(PythonParser):
                 self.addRule(
                     """
                     stmt        ::= assert_pypy
-                    stmt        ::= assert2_pypy", nop_func)
+                    stmt        ::= assert_not_pypy
+                    stmt        ::= assert2_pypy
+                    stmt        ::= assert2_not_pypy
                     assert_pypy ::=  JUMP_IF_NOT_DEBUG assert_expr jmp_true
                                      LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
+                    assert_not_pypy ::=  JUMP_IF_NOT_DEBUG assert_expr jmp_false
+                                     LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
                     assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true
                                      LOAD_ASSERT expr CALL_FUNCTION_1
                                      RAISE_VARARGS_1 COME_FROM
                     assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true
                                      LOAD_ASSERT expr CALL_FUNCTION_1
-                                     RAISE_VARARGS_1 COME_FROM,
+                                     RAISE_VARARGS_1 COME_FROM
+                    assert2_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false
+                                     LOAD_ASSERT expr CALL_FUNCTION_1
+                                     RAISE_VARARGS_1 COME_FROM
                     """,
                     nop_func,
                 )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_BUILD_CLASS":
-                self.custom_build_class_rule(opname, i, token, tokens, customize)
+                self.custom_build_class_rule(
+                    opname, i, token, tokens, customize, self.is_pypy
+                )
                 # Note: don't add to custom_ops_processed.
             elif opname == "LOAD_CLASSDEREF":
                 # Python 3.4+
@@ -967,7 +999,7 @@ class Python3Parser(PythonParser):
                     j = 1
                 else:
                     j = 2
-                if is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"):
+                if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"):
                     rule_pat = "mklambda ::= %sload_closure LOAD_LAMBDA %%s%s" % (
                         "pos_arg " * args_pos,
                         opname,
@@ -982,7 +1014,7 @@ class Python3Parser(PythonParser):
                     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
                     if has_get_iter_call_function1:
-                        if is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"):
+                        if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"):
                             # In the tokens we saw:
                             #   LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or
                             #   LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or
@@ -996,7 +1028,7 @@ class Python3Parser(PythonParser):
                             self.add_make_function_rule(
                                 rule_pat, opname, token.attr, customize
                             )
-                        if is_pypy or (i >= j and tokens[i - j] == "LOAD_SETCOMP"):
+                        if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_SETCOMP"):
                             rule_pat = (
                                 "set_comp ::= %sload_closure LOAD_SETCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
@@ -1005,7 +1037,7 @@ class Python3Parser(PythonParser):
                             self.add_make_function_rule(
                                 rule_pat, opname, token.attr, customize
                             )
-                        if is_pypy or (i >= j and tokens[i - j] == "LOAD_DICTCOMP"):
+                        if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_DICTCOMP"):
                             self.add_unique_rule(
                                 "dict_comp ::= %sload_closure LOAD_DICTCOMP %s "
                                 "expr GET_ITER CALL_FUNCTION_1"
@@ -1051,17 +1083,24 @@ class Python3Parser(PythonParser):
                         )
 
                 elif self.version >= 3.4:
+                    if PYTHON3 or not self.is_pypy:
+                        load_op = "LOAD_STR"
+                    else:
+                        load_op = "LOAD_CONST"
+
                     if annotate_args > 0:
-                        rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" % (
+                        rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure %s %s %s" % (
                             "pos_arg " * args_pos,
                             kwargs_str,
                             "annotate_arg " * (annotate_args - 1),
+                            load_op,
                             opname,
                         )
                     else:
-                        rule = "mkfunc ::= %s%s load_closure LOAD_CODE LOAD_STR %s" % (
+                        rule = "mkfunc ::= %s%s load_closure LOAD_CODE %s %s" % (
                             "pos_arg " * args_pos,
                             kwargs_str,
+                            load_op,
                             opname,
                         )
 
@@ -1119,6 +1158,14 @@ class Python3Parser(PythonParser):
                         opname,
                     )
                     self.add_unique_rule(rule, opname, token.attr, customize)
+                    if not PYTHON3 and self.is_pypy:
+                        rule = "mkfunc ::= %s%s%s%s" % (
+                            "expr " * stack_count,
+                            "load_closure " * closure,
+                            "LOAD_CODE LOAD_CONST ",
+                            opname,
+                        )
+                        self.add_unique_rule(rule, opname, token.attr, customize)
 
                     if has_get_iter_call_function1:
                         rule_pat = (
@@ -1135,7 +1182,7 @@ class Python3Parser(PythonParser):
                         self.add_make_function_rule(
                             rule_pat, opname, token.attr, customize
                         )
-                        if is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP"):
+                        if self.is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP"):
                             if self.version >= 3.6:
                                 # 3.6+ sometimes bundles all of the
                                 # 'exprs' in the rule above into a
@@ -1156,7 +1203,7 @@ class Python3Parser(PythonParser):
                                 rule_pat, opname, token.attr, customize
                             )
 
-                    if is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LAMBDA"):
+                    if self.is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LAMBDA"):
                         rule_pat = "mklambda ::= %s%sLOAD_LAMBDA %%s%s" % (
                             ("pos_arg " * args_pos),
                             ("kwarg " * args_kw),
@@ -1184,7 +1231,7 @@ class Python3Parser(PythonParser):
                     )
                     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
-                    if is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"):
+                    if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"):
                         # In the tokens we saw:
                         #   LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or
                         #   LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or
@@ -1199,7 +1246,7 @@ class Python3Parser(PythonParser):
                         )
 
                 # FIXME: Fold test  into add_make_function_rule
-                if is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"):
+                if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"):
                     rule_pat = "mklambda ::= %s%sLOAD_LAMBDA %%s%s" % (
                         ("pos_arg " * args_pos),
                         ("kwarg " * args_kw),
@@ -1390,8 +1437,9 @@ class Python3Parser(PythonParser):
                                          except_handler COME_FROM else_suitel
                                          opt_come_from_except
                     """,
-                    nop_func,
+                    nop_func
                 )
+
                 custom_ops_processed.add(opname)
             elif opname_base in ("UNPACK_EX",):
                 before_count, after_count = token.attr
@@ -1494,12 +1542,24 @@ class Python3Parser(PythonParser):
             for i in range(cfl - 1, first, -1):
                 if tokens[i] != "POP_BLOCK":
                     break
-            if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"):
+            if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "BREAK_LOOP"):
                 if not tokens[i].kind.startswith("COME_FROM"):
                     return True
 
             # Check that the SETUP_LOOP jumps to the offset after the
             # COME_FROM_LOOP
+
+            # Python 3.0 has additional:
+            #     JUMP_FORWARD here
+            #     COME_FROM
+            #     POP_TOP
+            #     COME_FROM
+            #  here:
+            #     (target of SETUP_LOOP)
+            # We won't check this.
+            if self.version == 3.0:
+                return False
+
             if 0 <= last < len(tokens) and tokens[last] in (
                 "COME_FROM_LOOP",
                 "JUMP_BACK",

uncompyle6/parsers/parse30.py

@@ -12,13 +12,19 @@ class Python30Parser(Python31Parser):
     def p_30(self, args):
         """
 
+        pt_bp             ::= POP_TOP POP_BLOCK
+
         assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP
+        assert2           ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
+                              come_froms
+        call_stmt         ::= expr _come_froms POP_TOP
+
         return_if_lambda  ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP
         compare_chained2  ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
 
         # FIXME: combine with parse3.2
-        whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK
-                              COME_FROM_LOOP
+        whileTruestmt     ::= SETUP_LOOP l_stmts_opt
+                              jb_or_c COME_FROM_LOOP
         whileTruestmt     ::= SETUP_LOOP returns
                               COME_FROM_LOOP
 
@@ -43,10 +49,22 @@ class Python30Parser(Python31Parser):
 
         else_suitel ::= l_stmts COME_FROM_LOOP JUMP_BACK
 
-        ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel
+        jump_absolute_else ::= COME_FROM JUMP_ABSOLUTE COME_FROM POP_TOP
+
+        jump_cf_pop   ::= _come_froms _jump  _come_froms POP_TOP
+
+        ifelsestmt  ::= testexpr c_stmts_opt jump_cf_pop else_suite COME_FROM
+        ifelsestmtl ::= testexpr c_stmts_opt jump_cf_pop else_suitel
+        ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
+        ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec
+
+        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM
         iflaststmtl ::= testexpr c_stmts_opt jb_pop_top
+        iflaststmtl ::= testexpr c_stmts_opt come_froms JUMP_BACK COME_FROM POP_TOP
+
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
 
+
         withasstmt    ::= expr setupwithas store suite_stmts_opt
                           POP_BLOCK LOAD_CONST COME_FROM_FINALLY
                           LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
@@ -54,8 +72,8 @@ class Python30Parser(Python31Parser):
         setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
 
         # Need to keep LOAD_FAST as index 1
-        set_comp_func_header ::=  BUILD_SET_0 DUP_TOP STORE_FAST
-        set_comp_func ::= set_comp_func_header
+        set_comp_header  ::= BUILD_SET_0 DUP_TOP STORE_FAST
+        set_comp_func ::= set_comp_header
                           LOAD_FAST FOR_ITER store comp_iter
                           JUMP_BACK POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST
 
@@ -63,8 +81,10 @@ class Python30Parser(Python31Parser):
         list_comp        ::= list_comp_header
                              LOAD_FAST FOR_ITER store comp_iter
                              JUMP_BACK
+        list_comp        ::= list_comp_header
+                             LOAD_FAST FOR_ITER store comp_iter
+                             JUMP_BACK _come_froms POP_TOP JUMP_BACK
 
-        set_comp_header  ::= BUILD_SET_0 DUP_TOP STORE_FAST
         set_comp         ::= set_comp_header
                              LOAD_FAST FOR_ITER store comp_iter
                              JUMP_BACK
@@ -73,6 +93,24 @@ class Python30Parser(Python31Parser):
         dict_comp        ::= dict_comp_header
                              LOAD_FAST FOR_ITER store dict_comp_iter
                              JUMP_BACK
+        dict_comp        ::= dict_comp_header
+                             LOAD_FAST FOR_ITER store dict_comp_iter
+                             JUMP_BACK _come_froms POP_TOP JUMP_BACK
+
+        stmt         ::= try_except30
+        try_except30 ::= SETUP_EXCEPT suite_stmts_opt
+                        _come_froms pt_bp
+                         except_handler opt_come_from_except
+
+        # From Python 2.6
+
+
+        list_iter  ::= list_if JUMP_BACK
+        list_iter  ::= list_if JUMP_BACK _come_froms POP_TOP
+        lc_body    ::= LOAD_NAME expr LIST_APPEND
+	lc_body    ::= LOAD_FAST expr LIST_APPEND
+        list_if    ::= expr jmp_false_then list_iter
+        #############
 
         dict_comp_iter   ::= expr expr ROT_TWO expr STORE_SUBSCR
 
@@ -88,19 +126,52 @@ class Python30Parser(Python31Parser):
         except_suite          ::= c_stmts POP_EXCEPT jump_except POP_TOP
         except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY
                                   _jump COME_FROM POP_TOP
-        jump_except           ::= JUMP_FORWARD COME_FROM POP_TOP
-        jump_except           ::= JUMP_ABSOLUTE COME_FROM POP_TOP
+
+        except_handler        ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY
+
+        _ifstmts_jump         ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
+        _ifstmts_jump         ::= c_stmts_opt come_froms POP_TOP JUMP_FORWARD _come_froms
+
+        jump_except           ::= _jump COME_FROM POP_TOP
+
         or                    ::= expr jmp_false expr jmp_true expr
         or                    ::= expr jmp_true expr
 
+        import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist _come_froms POP_TOP
+
         ################################################################################
         # In many ways 3.0 is like 2.6. One similarity is there is no JUMP_IF_TRUE and
         # JUMP_IF_FALSE
         # The below rules in fact are the same or similar.
 
-        jmp_true       ::= JUMP_IF_TRUE POP_TOP
-        jmp_false      ::= JUMP_IF_FALSE _come_froms POP_TOP
+        jmp_true         ::= JUMP_IF_TRUE POP_TOP
+        jmp_true_then    ::= JUMP_IF_TRUE _come_froms POP_TOP
+        jmp_false        ::= JUMP_IF_FALSE _come_froms POP_TOP
+        jmp_false_then   ::= JUMP_IF_FALSE POP_TOP
 
+        # We don't have hacky THEN detection, so we do it
+        # in the grammar below which is also somewhat hacky.
+
+        stmt             ::= ifstmt30
+        stmt             ::= ifnotstmt30
+        ifstmt30         ::= testfalse_then _ifstmts_jump30
+        ifnotstmt30      ::= testtrue_then  _ifstmts_jump30
+
+        testfalse_then   ::= expr jmp_false_then
+        testtrue_then    ::= expr jmp_true_then
+        call_stmt        ::= expr COME_FROM
+        _ifstmts_jump30  ::= c_stmts POP_TOP
+
+        gen_comp_body    ::= expr YIELD_VALUE COME_FROM POP_TOP
+
+        except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
+                             COME_FROM POP_TOP END_FINALLY
+
+        or               ::= expr jmp_true_then expr come_from_opt
+        ret_or           ::= expr jmp_true_then expr come_from_opt
+        ret_and          ::= expr jump_false expr come_from_opt
+
+        ################################################################################
         for_block      ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
 
         except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
@@ -108,12 +179,17 @@ class Python30Parser(Python31Parser):
         except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
                            POP_TOP END_FINALLY
 
-        return_if_stmt ::= ret_expr RETURN_END_IF COME_FROM POP_TOP
-        and            ::= expr jmp_false expr come_from_opt
+        return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP
+        return_if_stmt ::= ret_expr RETURN_VALUE come_froms POP_TOP
+
+        and            ::= expr jmp_false_then expr come_from_opt
+
         whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
-                           JUMP_BACK COME_FROM POP_TOP POP_BLOCK COME_FROM_LOOP
+                           JUMP_BACK _come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
         whilestmt      ::= SETUP_LOOP testexpr returns
                            POP_TOP POP_BLOCK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
+                           come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
 
 
         # compare_chained is like x <= y <= z
@@ -124,34 +200,162 @@ class Python30Parser(Python31Parser):
         compare_chained2 ::= expr COMPARE_OP RETURN_END_IF
         """
 
-    def customize_grammar_rules(self, tokens, customize):
-        super(Python30Parser, self).customize_grammar_rules(tokens, customize)
+
+    def remove_rules_30(self):
         self.remove_rules("""
+
+        # The were found using grammar coverage
+        while1stmt     ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+        whileTruestmt  ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
+        whileelsestmt  ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP
+        withasstmt     ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
+        withstmt       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
+        ##########################################################################################
+
         iflaststmtl        ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
         ifelsestmtl        ::= testexpr c_stmts_opt JUMP_BACK else_suitel
         iflaststmt         ::= testexpr c_stmts_opt JUMP_ABSOLUTE
         _ifstmts_jump      ::= c_stmts_opt JUMP_FORWARD _come_froms
+
         jump_forward_else  ::= JUMP_FORWARD ELSE
         jump_absolute_else ::= JUMP_ABSOLUTE ELSE
         whilestmt          ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
                                COME_FROM_LOOP
         whilestmt          ::= SETUP_LOOP testexpr returns
                                POP_BLOCK COME_FROM_LOOP
+
         assert             ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
+
         return_if_lambda   ::= RETURN_END_IF_LAMBDA
         except_suite       ::= c_stmts POP_EXCEPT jump_except
         whileelsestmt      ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK
                                else_suitel COME_FROM_LOOP
 
-        # No JUMP_IF_FALSE_OR_POP
+        ################################################################
+        # No JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP,
+        # POP_JUMP_IF_FALSE, or POP_JUMP_IF_TRUE
+
+        jmp_false        ::= POP_JUMP_IF_FALSE
+        jmp_true         ::= JUMP_IF_TRUE_OR_POP POP_TOP
+        jmp_true         ::= POP_JUMP_IF_TRUE
+
         compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
                              compare_chained1 COME_FROM
         compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
                              compare_chained2 COME_FROM
+        ret_or           ::= expr JUMP_IF_TRUE_OR_POP  ret_expr_or_cond COME_FROM
+        ret_and          ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
+        ret_cond         ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF
+                             COME_FROM ret_expr_or_cond
+        ret_expr_or_cond ::= ret_cond
+        or               ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
+        and              ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
+        and              ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
         """)
 
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python30Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules_30()
+
+        self.check_reduce["iflaststmtl"] = "AST"
+        self.check_reduce['ifstmt'] = "AST"
+        self.check_reduce["ifelsestmtc"] = "AST"
+        self.check_reduce["ifelsestmt"] = "AST"
+        # self.check_reduce["and"] = "stmt"
         return
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python30Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        lhs = rule[0]
+        if (
+            lhs in ("iflaststmtl", "ifstmt",
+                        "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr"
+        ):
+            testexpr = ast[0]
+            if testexpr[0] == "testfalse":
+                testfalse = testexpr[0]
+                if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else":
+                    jump_absolute_else = ast[2]
+                    come_from = jump_absolute_else[2]
+                    return come_from == "COME_FROM" and come_from.attr < tokens[first].offset
+                    pass
+                elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop":
+                    jump_cf_pop = ast[2]
+                    come_froms = jump_cf_pop[0]
+                    for come_from in come_froms:
+                        if come_from.attr < tokens[first].offset:
+                            return True
+                    come_froms = jump_cf_pop[2]
+                    if come_froms == "COME_FROM":
+                        if come_froms.attr < tokens[first].offset:
+                            return True
+                        pass
+                    elif come_froms == "_come_froms":
+                        for come_from in come_froms:
+                            if come_from.attr < tokens[first].offset:
+                                return True
+
+                    return False
+                elif testfalse[1] == "jmp_false":
+                    jmp_false = testfalse[1]
+                    if last == len(tokens):
+                        last -= 1
+                    while (isinstance(tokens[first].offset, str) and first < last):
+                        first += 1
+                    if first == last:
+                        return True
+                    while (first < last and isinstance(tokens[last].offset, str)):
+                        last -= 1
+                    if rule[0] == "iflaststmtl":
+                        return not (jmp_false[0].attr <= tokens[last].offset)
+                    else:
+                        jmp_false_target = jmp_false[0].attr
+                        if tokens[first].offset > jmp_false_target:
+                            return True
+                        return (
+                            (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD")
+                    pass
+                pass
+            pass
+        # elif lhs == "and":
+        #     return tokens[last+1] == "JUMP_FORWARD"
+
     pass
 
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python30Parser()
+    p.remove_rules_30()
+    p.check_grammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.0:
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        ## FIXME: try this
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub(r'_\d+$', '',  t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/parse31.py

@@ -32,11 +32,45 @@ class Python31Parser(Python32Parser):
         load  ::= LOAD_FAST
         load  ::= LOAD_NAME
         """
+    def remove_rules_31(self):
+        self.remove_rules("""
+        # DUP_TOP_TWO is DUP_TOPX in 3.1 and earlier
+        subscript2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
+        """)
 
     def customize_grammar_rules(self, tokens, customize):
         super(Python31Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules_31()
         return
     pass
 
 class Python31ParserSingle(Python31Parser, PythonParserSingle):
     pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python31Parser()
+    p.remove_rules_31()
+    p.check_grammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.1:
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        ## FIXME: try this
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub(r'_\d+$', '',  t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/parse32.py

@@ -8,9 +8,15 @@ from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse3 import Python3Parser
 
 class Python32Parser(Python3Parser):
+    def p_30to33(self, args):
+        """
+        # Store locals is only in Python 3.0 to 3.3
+        stmt           ::= store_locals
+        store_locals   ::= LOAD_FAST STORE_LOCALS
+        """
+
     def p_32to35(self, args):
         """
-        expr        ::= conditional
         conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM
 
         # compare_chained2 is used in a "chained_compare": x <= y <= z
@@ -18,10 +24,6 @@ class Python32Parser(Python3Parser):
         compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
         compare_chained2 ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
 
-        # Store locals is only in Python 3.0 to 3.3
-        stmt           ::= store_locals
-        store_locals   ::= LOAD_FAST STORE_LOCALS
-
         # Python < 3.5 no POP BLOCK
         whileTruestmt  ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM_LOOP
 

uncompyle6/parsers/parse33.py

@@ -14,23 +14,6 @@ class Python33Parser(Python32Parser):
         # Python 3.3+ adds yield from.
         expr          ::= yield_from
         yield_from    ::= expr expr YIELD_FROM
-
-        # We do the grammar hackery below for semantics
-        # actions that want c_stmts_opt at index 1
-
-        # Python 3.5+ has jump optimization to remove the redundant
-        # jump_excepts. But in 3.3 we need them added
-
-        try_except  ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
-                        except_handler
-                        jump_excepts come_from_except_clauses
-        """
-
-    def p_30to33(self, args):
-        """
-        # Store locals is only in Python 3.0 to 3.3
-        stmt           ::= store_locals
-        store_locals   ::= LOAD_FAST STORE_LOCALS
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/parse34.py

@@ -72,7 +72,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.4:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse35.py

@@ -144,9 +144,15 @@ class Python35Parser(Python34Parser):
         super(Python35Parser, self).customize_grammar_rules(tokens, customize)
         for i, token in enumerate(tokens):
             opname = token.kind
+            if opname == 'LOAD_ASSERT':
+                if 'PyPy' in customize:
+                    rules_str = """
+                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
+                    """
+                    self.add_unique_doc_rules(rules_str, customize)
             # FIXME: I suspect this is wrong for 3.6 and 3.5, but
             # I haven't verified what the 3.7ish fix is
-            if opname == 'BUILD_MAP_UNPACK_WITH_CALL':
+            elif opname == 'BUILD_MAP_UNPACK_WITH_CALL':
                 if self.version < 3.7:
                     self.addRule("expr ::= unmapexpr", nop_func)
                     nargs = token.attr % 256
@@ -257,7 +263,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.5:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse36.py

@@ -188,13 +188,7 @@ class Python36Parser(Python35Parser):
         for i, token in enumerate(tokens):
             opname = token.kind
 
-            if opname == 'LOAD_ASSERT':
-                if 'PyPy' in customize:
-                    rules_str = """
-                    stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
-                    """
-                    self.add_unique_doc_rules(rules_str, customize)
-            elif opname == 'FORMAT_VALUE':
+            if opname == 'FORMAT_VALUE':
                 rules_str = """
                     expr              ::= formatted_value1
                     formatted_value1  ::= expr FORMAT_VALUE
@@ -316,7 +310,7 @@ class Python36Parser(Python35Parser):
             pass
         return
 
-    def custom_classfunc_rule(self, opname, token, customize, next_token):
+    def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
 
         args_pos, args_kw = self.get_pos_kw(token)
 
@@ -338,10 +332,14 @@ class Python36Parser(Python35Parser):
             self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
 
         if opname.startswith('CALL_FUNCTION_KW'):
-            self.addRule("expr ::= call_kw36", nop_func)
-            values = 'expr ' * token.attr
-            rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
-            self.add_unique_rule(rule, token.kind, token.attr, customize)
+            if is_pypy:
+                # PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions
+                super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
+            else:
+                self.addRule("expr ::= call_kw36", nop_func)
+                values = 'expr ' * token.attr
+                rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
+                self.add_unique_rule(rule, token.kind, token.attr, customize)
         elif opname == 'CALL_FUNCTION_EX_KW':
             # Note: this doesn't exist in 3.7 and later
             self.addRule("""expr        ::= call_ex_kw4
@@ -406,7 +404,7 @@ class Python36Parser(Python35Parser):
                             """, nop_func)
             pass
         else:
-            super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token)
+            super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         invalid = super(Python36Parser,
@@ -443,7 +441,7 @@ if __name__ == '__main__':
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
     if PYTHON_VERSION == 3.6:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(

uncompyle6/parsers/parse37.py

@@ -141,31 +141,6 @@ class Python37Parser(Python36Parser):
         """
 
     def customize_grammar_rules(self, tokens, customize):
-        self.remove_rules("""
-          async_forelse_stmt ::= SETUP_LOOP expr
-                                 GET_AITER
-                                 LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
-                                 YIELD_FROM
-                                 store
-                                 POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
-                                 LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
-                                 POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
-                                 JUMP_ABSOLUTE END_FINALLY COME_FROM
-                                 for_block POP_BLOCK
-                                 else_suite COME_FROM_LOOP
-        stmt      ::= async_for_stmt36
-        async_for_stmt36   ::= SETUP_LOOP expr
-                               GET_AITER
-                               LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
-                               YIELD_FROM
-                               store
-                               POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
-                               LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
-                               END_FINALLY continues COME_FROM
-                               POP_TOP POP_TOP POP_TOP POP_EXCEPT
-                               POP_TOP POP_BLOCK
-                               COME_FROM_LOOP
-        """)
         super(Python37Parser, self).customize_grammar_rules(tokens, customize)
 
 class Python37ParserSingle(Python37Parser, PythonParserSingle):
@@ -173,22 +148,33 @@ class Python37ParserSingle(Python37Parser, PythonParserSingle):
 
 if __name__ == '__main__':
     # Check grammar
+    # FIXME: DRY this with other parseXX.py routines
     p = Python37Parser()
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
+
     if PYTHON_VERSION == 3.7:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+        opcode_set = set(s.opc.opname).union(
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split()
+            )
+        )
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        # print(sorted(p.rule2name.items()))
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/parse38.py

@@ -92,6 +92,7 @@ class Python38Parser(Python37Parser):
         for38              ::= expr get_iter store for_block JUMP_BACK
         for38              ::= expr for_iter store for_block JUMP_BACK
         for38              ::= expr for_iter store for_block JUMP_BACK POP_BLOCK
+        for38              ::= expr for_iter store for_block
 
         forelsestmt38      ::= expr for_iter store for_block POP_BLOCK else_suite
         forelselaststmt38  ::= expr for_iter store for_block POP_BLOCK else_suitec
@@ -158,7 +159,7 @@ class Python38Parser(Python37Parser):
         super(Python38Parser, self).__init__(debug_parser)
         self.customized = {}
 
-    def customize_grammar_rules(self, tokens, customize):
+    def remove_rules_38(self):
         self.remove_rules("""
            stmt               ::= async_for_stmt37
            stmt               ::= for
@@ -225,7 +226,10 @@ class Python38Parser(Python37Parser):
 
 
         """)
+
+    def customize_grammar_rules(self, tokens, customize):
         super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules_38()
         self.check_reduce['ifstmt'] = 'tokens'
         self.check_reduce['whileTruestmt38'] = 'tokens'
 
@@ -233,6 +237,7 @@ class Python38Parser(Python37Parser):
         invalid = super(Python38Parser,
                         self).reduce_is_invalid(rule, ast,
                                                 tokens, first, last)
+        self.remove_rules_38()
         if invalid:
             return invalid
         if rule[0] == 'ifstmt':
@@ -263,24 +268,34 @@ class Python38Parser(Python37Parser):
 class Python38ParserSingle(Python38Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     # Check grammar
+    # FIXME: DRY this with other parseXX.py routines
     p = Python38Parser()
+    p.remove_rules_38()
     p.check_grammar()
     from uncompyle6 import PYTHON_VERSION, IS_PYPY
+
     if PYTHON_VERSION == 3.8:
-        lhs, rhs, tokens, right_recursive = p.check_sets()
+        lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
-               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
-               LAMBDA_MARKER RETURN_LAST
+        opcode_set = set(s.opc.opname).union(
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+                LAMBDA_MARKER RETURN_LAST
             """.split()))
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
-        # print(sorted(p.rule2name.items()))
+        import sys
+        if len(sys.argv) > 1:
+            from spark_parser.spark import rule2str
+            for rule in sorted(p.rule2name.items()):
+                print(rule2str(rule[0]))

uncompyle6/parsers/treenode.py

@@ -54,3 +54,11 @@ class SyntaxTree(spark_AST):
             rv += "\n" + child
             i += 1
         return rv
+
+    def first_child(self):
+        if len(self) > 0:
+            child = self[0]
+            if not isinstance(child, SyntaxTree):
+                return child
+            return self[0].first_child()
+        return self

uncompyle6/scanner.py

@@ -30,22 +30,44 @@ import sys
 from uncompyle6 import PYTHON3, IS_PYPY
 from uncompyle6.scanners.tok import Token
 import xdis
-from xdis.bytecode import (
-    Bytecode, instruction_size, extended_arg_val, next_offset)
+from xdis.bytecode import Bytecode, instruction_size, extended_arg_val, next_offset
 from xdis.magics import canonic_python_version
 from xdis.util import code2num
 
 # The byte code versions we support.
 # Note: these all have to be floats
-PYTHON_VERSIONS = frozenset((1.0, 1.1, 1.3, 1.4, 1.5, 1.6,
-                             2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
-                             3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8))
+PYTHON_VERSIONS = frozenset(
+    (
+        1.0,
+        1.1,
+        1.3,
+        1.4,
+        1.5,
+        1.6,
+        2.1,
+        2.2,
+        2.3,
+        2.4,
+        2.5,
+        2.6,
+        2.7,
+        3.0,
+        3.1,
+        3.2,
+        3.3,
+        3.4,
+        3.5,
+        3.6,
+        3.7,
+        3.8,
+    )
+)
 
 CANONIC2VERSION = dict((canonic_python_version[str(v)], v) for v in PYTHON_VERSIONS)
 
 # Magic changed mid version for Python 3.5.2. Compatibility was added for
 # the older 3.5 interpreter magic.
-CANONIC2VERSION['3.5.2'] = 3.5
+CANONIC2VERSION["3.5.2"] = 3.5
 
 
 # FIXME: DRY
@@ -55,24 +77,28 @@ if PYTHON3:
 
     def long(l):
         return l
+
+
 else:
-    L65536 = long(65536) # NOQA
+    L65536 = long(65536)  # NOQA
+
 
 class Code(object):
-    '''
+    """
     Class for representing code-objects.
 
     This is similar to the original code object, but additionally
     the diassembled code is stored in the attribute '_tokens'.
-    '''
+    """
+
     def __init__(self, co, scanner, classname=None):
         for i in dir(co):
-            if i.startswith('co_'):
+            if i.startswith("co_"):
                 setattr(self, i, getattr(co, i))
         self._tokens, self._customize = scanner.ingest(co, classname)
 
-class Scanner(object):
 
+class Scanner(object):
     def __init__(self, version, show_asm=None, is_pypy=False):
         self.version = version
         self.show_asm = show_asm
@@ -100,7 +126,7 @@ class Scanner(object):
         """
         # FIXME: remove this when all subsidiary functions have been removed.
         # We should be able to get everything from the self.insts list.
-        self.code = array('B', co.co_code)
+        self.code = array("B", co.co_code)
 
         bytecode = Bytecode(co, self.opc)
         self.build_prev_op()
@@ -128,7 +154,7 @@ class Scanner(object):
         # 'List-map' which shows line number of current op and offset of
         # first op on following line, given offset of op as index
         lines = []
-        LineTuple = namedtuple('LineTuple', ['l_no', 'next'])
+        LineTuple = namedtuple("LineTuple", ["l_no", "next"])
 
         # Iterate through available linestarts, and fill
         # the data for all code offsets encountered until
@@ -171,14 +197,14 @@ class Scanner(object):
         goes forward.
         """
         opname = self.get_inst(offset).opname
-        if opname == 'JUMP_FORWARD':
+        if opname == "JUMP_FORWARD":
             return True
-        if opname != 'JUMP_ABSOLUTE':
+        if opname != "JUMP_ABSOLUTE":
             return False
         return offset < self.get_target(offset)
 
     def prev_offset(self, offset):
-        return self.insts[self.offset2inst_index[offset]-1].offset
+        return self.insts[self.offset2inst_index[offset] - 1].offset
 
     def get_inst(self, offset):
         # Instructions can get moved as a result of EXTENDED_ARGS removal.
@@ -205,7 +231,7 @@ class Scanner(object):
         return target
 
     def get_argument(self, pos):
-        arg = self.code[pos+1] + self.code[pos+2] * 256
+        arg = self.code[pos + 1] + self.code[pos + 2] * 256
         return arg
 
     def next_offset(self, op, offset):
@@ -216,9 +242,9 @@ class Scanner(object):
             op = self.code[i]
             if op in self.JUMP_OPS:
                 dest = self.get_target(i, op)
-                print('%i\t%s\t%i' % (i, self.opname[op], dest))
+                print("%i\t%s\t%i" % (i, self.opname[op], dest))
             else:
-                print('%i\t%s\t' % (i, self.opname[op]))
+                print("%i\t%s\t" % (i, self.opname[op]))
 
     def first_instr(self, start, end, instr, target=None, exact=True):
         """
@@ -232,11 +258,9 @@ class Scanner(object):
         Return index to it or None if not found.
         """
         code = self.code
-        assert(start >= 0 and end <= len(code))
+        assert start >= 0 and end <= len(code)
 
-        try:
-            None in instr
-        except:
+        if not isinstance(instr, list):
             instr = [instr]
 
         result_offset = None
@@ -274,9 +298,7 @@ class Scanner(object):
         if not (start >= 0 and end <= len(code)):
             return None
 
-        try:
-            None in instr
-        except:
+        if not isinstance(instr, list):
             instr = [instr]
 
         result_offset = None
@@ -287,7 +309,7 @@ class Scanner(object):
             op = code[offset]
 
             if op == self.opc.EXTENDED_ARG:
-                arg = code2num(code, offset+1) | extended_arg
+                arg = code2num(code, offset + 1) | extended_arg
                 extended_arg = extended_arg_val(self.opc, arg)
                 continue
 
@@ -365,7 +387,7 @@ class Scanner(object):
         """
 
         code = self.code
-        assert(start >= 0 and end <= len(code))
+        assert start >= 0 and end <= len(code)
 
         try:
             None in instr
@@ -379,7 +401,7 @@ class Scanner(object):
             op = code[offset]
 
             if op == self.opc.EXTENDED_ARG:
-                arg = code2num(code, offset+1) | extended_arg
+                arg = code2num(code, offset + 1) | extended_arg
                 extended_arg = extended_arg_val(self.opc, arg)
                 continue
 
@@ -423,8 +445,11 @@ class Scanner(object):
         last_was_extarg = False
         n = len(instructions)
         for i, inst in enumerate(instructions):
-            if (inst.opname == 'EXTENDED_ARG'
-                and i+1 < n and instructions[i+1].opname != 'MAKE_FUNCTION'):
+            if (
+                inst.opname == "EXTENDED_ARG"
+                and i + 1 < n
+                and instructions[i + 1].opname != "MAKE_FUNCTION"
+            ):
                 last_was_extarg = True
                 starts_line = inst.starts_line
                 is_jump_target = inst.is_jump_target
@@ -435,13 +460,15 @@ class Scanner(object):
                 # j = self.stmts.index(inst.offset)
                 # self.lines[j] = offset
 
-                new_inst = inst._replace(starts_line=starts_line,
-                                        is_jump_target=is_jump_target,
-                                        offset=offset)
+                new_inst = inst._replace(
+                    starts_line=starts_line,
+                    is_jump_target=is_jump_target,
+                    offset=offset,
+                )
                 inst = new_inst
                 if i < n:
                     new_prev = self.prev_op[instructions[i].offset]
-                    j = instructions[i+1].offset
+                    j = instructions[i + 1].offset
                     old_prev = self.prev_op[j]
                     while self.prev_op[j] == old_prev and j < n:
                         self.prev_op[j] = new_prev
@@ -463,9 +490,12 @@ class Scanner(object):
         for i in ifs:
             # For each offset, if line number of current and next op
             # is the same
-            if self.lines[i].l_no == self.lines[i+3].l_no:
+            if self.lines[i].l_no == self.lines[i + 3].l_no:
                 # Skip last op on line if it is some sort of POP_JUMP.
-                if self.code[self.prev[self.lines[i].next]] in (self.opc.PJIT, self.opc.PJIF):
+                if self.code[self.prev[self.lines[i].next]] in (
+                    self.opc.PJIT,
+                    self.opc.PJIF,
+                ):
                     continue
             filtered.append(i)
         return filtered
@@ -475,8 +505,8 @@ class Scanner(object):
 
     def restrict_to_parent(self, target, parent):
         """Restrict target to parent structure boundaries."""
-        if not (parent['start'] < target < parent['end']):
-            target = parent['end']
+        if not (parent["start"] < target < parent["end"]):
+            target = parent["end"]
         return target
 
     def setTokenClass(self, tokenClass):
@@ -484,6 +514,7 @@ class Scanner(object):
         self.Token = tokenClass
         return self.Token
 
+
 def parse_fn_counts(argc):
     return ((argc & 0xFF), (argc >> 8) & 0xFF, (argc >> 16) & 0x7FFF)
 
@@ -496,8 +527,10 @@ def get_scanner(version, is_pypy=False, show_asm=None):
             raise RuntimeError("Unknown Python version in xdis %s" % version)
         canonic_version = canonic_python_version[version]
         if canonic_version not in CANONIC2VERSION:
-            raise RuntimeError("Unsupported Python version %s (canonic %s)"
-                               % (version, canonic_version))
+            raise RuntimeError(
+                "Unsupported Python version %s (canonic %s)"
+                % (version, canonic_version)
+            )
         version = CANONIC2VERSION[canonic_version]
 
     # Pick up appropriate scanner
@@ -505,24 +538,34 @@ def get_scanner(version, is_pypy=False, show_asm=None):
         v_str = "%s" % (int(version * 10))
         try:
             import importlib
+
             if is_pypy:
                 scan = importlib.import_module("uncompyle6.scanners.pypy%s" % v_str)
             else:
                 scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str)
-            if False: print(scan)  # Avoid unused scan
+            if False:
+                print(scan)  # Avoid unused scan
         except ImportError:
             if is_pypy:
-                exec("import uncompyle6.scanners.pypy%s as scan" % v_str,
-                     locals(), globals())
+                exec(
+                    "import uncompyle6.scanners.pypy%s as scan" % v_str,
+                    locals(),
+                    globals(),
+                )
             else:
-                exec("import uncompyle6.scanners.scanner%s as scan" % v_str,
-                      locals(), globals())
+                exec(
+                    "import uncompyle6.scanners.scanner%s as scan" % v_str,
+                    locals(),
+                    globals(),
+                )
         if is_pypy:
-            scanner = eval("scan.ScannerPyPy%s(show_asm=show_asm)" % v_str,
-                           locals(), globals())
+            scanner = eval(
+                "scan.ScannerPyPy%s(show_asm=show_asm)" % v_str, locals(), globals()
+            )
         else:
-            scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str,
-                           locals(), globals())
+            scanner = eval(
+                "scan.Scanner%s(show_asm=show_asm)" % v_str, locals(), globals()
+            )
     else:
         raise RuntimeError("Unsupported Python version %s" % version)
     return scanner
@@ -530,8 +573,9 @@ def get_scanner(version, is_pypy=False, show_asm=None):
 
 if __name__ == "__main__":
     import inspect, uncompyle6
+
     co = inspect.currentframe().f_code
     # scanner = get_scanner('2.7.13', True)
     # scanner = get_scanner(sys.version[:5], False)
     scanner = get_scanner(uncompyle6.PYTHON_VERSION, IS_PYPY, True)
-    tokens, customize = scanner.ingest(co, {}, show_asm='after')
+    tokens, customize = scanner.ingest(co, {}, show_asm="after")

uncompyle6/scanners/pypy32.py

@@ -9,8 +9,8 @@ make things easier for decompilation.
 import uncompyle6.scanners.scanner32 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
-from xdis.opcodes import opcode_32 as opc  # is this right?
-JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+from xdis.opcodes import opcode_32pypy as opc
+JUMP_OPs = opc.JUMP_OPS
 
 # We base this off of 3.2
 class ScannerPyPy32(scan.Scanner32):
@@ -19,4 +19,5 @@ class ScannerPyPy32(scan.Scanner32):
         # pypy 3.2 and 3.2
         scan.Scanner32.__init__(self, show_asm, is_pypy=True)
         self.version = 3.2
+        self.opc = opc
         return

uncompyle6/scanners/pypy33.py

@@ -0,0 +1,23 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+"""
+Python PyPy 3.3 decompiler scanner.
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+"""
+
+import uncompyle6.scanners.scanner33 as scan
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_33pypy as opc
+JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+
+# We base this off of 3.3
+class ScannerPyPy33(scan.Scanner33):
+    def __init__(self, show_asm):
+        # There are no differences in initialization between
+        # pypy 3.3 and 3.3
+        scan.Scanner33.__init__(self, show_asm, is_pypy=True)
+        self.version = 3.3
+        self.opc = opc
+        return

uncompyle6/scanners/scanner3.py

@@ -257,22 +257,31 @@ class Scanner3(Scanner):
             # RAISE_VARARGS then we have a "raise" statement
             # else we have an "assert" statement.
             if self.version == 3.0:
-                # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here
+                # Like 2.6, 3.0 doesn't have POP_JUMP_IF... so we have
+                # to go through more machinations
                 assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n
+                if assert_can_follow:
+                    prev_inst = self.insts[i - 1]
+                    assert_can_follow = (
+                        prev_inst.opname in ("JUMP_IF_TRUE", "JUMP_IF_FALSE")
+                        and i + 1 < n )
+                    jump_if_inst = prev_inst
             else:
-                assert_can_follow = inst.opname == "POP_JUMP_IF_TRUE" and i + 1 < n
+                assert_can_follow = (
+                    inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE")
+                    and i + 1 < n
+                )
+                jump_if_inst = inst
             if assert_can_follow:
                 next_inst = self.insts[i + 1]
                 if (
                     next_inst.opname == "LOAD_GLOBAL"
                     and next_inst.argval == "AssertionError"
-                    and inst.argval
+                    and jump_if_inst.argval
                 ):
-                    raise_idx = self.offset2inst_index[self.prev_op[inst.argval]]
+                    raise_idx = self.offset2inst_index[self.prev_op[jump_if_inst.argval]]
                     raise_inst = self.insts[raise_idx]
-                    if raise_inst.opname.startswith(
-                        "RAISE_VARARGS"
-                    ):
+                    if raise_inst.opname.startswith("RAISE_VARARGS"):
                         self.load_asserts.add(next_inst.offset)
                     pass
                 pass
@@ -428,11 +437,16 @@ class Scanner3(Scanner):
                 else:
                     opname = "%s_%d" % (opname, pos_args)
 
-            elif self.is_pypy and opname == "JUMP_IF_NOT_DEBUG":
-                # The value in the dict is in special cases in semantic actions, such
-                # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put
-                # in arbitrary value 0.
-                customize[opname] = 0
+            elif self.is_pypy and opname in ("JUMP_IF_NOT_DEBUG", "CALL_FUNCTION"):
+                if opname == "JUMP_IF_NOT_DEBUG":
+                    # The value in the dict is in special cases in semantic actions, such
+                    # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put
+                    # in arbitrary value 0.
+                    customize[opname] = 0
+                elif self.version >= 3.6 and argval > 255:
+                    opname = "CALL_FUNCTION_KW"
+                    pass
+
             elif opname == "UNPACK_EX":
                 # FIXME: try with scanner and parser by
                 # changing argval
@@ -468,6 +482,12 @@ class Scanner3(Scanner):
                         and self.insts[i + 1].opname == "JUMP_FORWARD"
                     )
 
+                    if (self.version == 3.0 and self.insts[i + 1].opname == "JUMP_FORWARD"
+                        and not is_continue):
+                        target_prev = self.offset2inst_index[self.prev_op[target]]
+                        is_continue = (
+                            self.insts[target_prev].opname == "SETUP_LOOP")
+
                     if is_continue or (
                         inst.offset in self.stmts
                         and (

uncompyle6/scanners/scanner33.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2018 by Rocky Bernstein
+#  Copyright (c) 2015-2019 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -28,7 +28,7 @@ JUMP_OPS = opc.JUMP_OPS
 from uncompyle6.scanners.scanner3 import Scanner3
 class Scanner33(Scanner3):
 
-    def __init__(self, show_asm=False):
+    def __init__(self, show_asm=False, is_pypy=False):
         Scanner3.__init__(self, 3.3, show_asm)
         return
     pass

uncompyle6/semantics/customize.py

@@ -28,8 +28,15 @@ def customize_for_version(self, is_pypy, version):
         # PyPy changes
         #######################
         TABLE_DIRECT.update({
-            'assert_pypy':	( '%|assert %c\n' , 1 ),
-            'assert2_pypy':	( '%|assert %c, %c\n' , 1, 4 ),
+            'assert_pypy':	( '%|assert %c\n' ,     (1, 'assert_expr') ),
+            # This is as a result of an if transoration
+            'assert0_pypy':	( '%|assert %c\n' ,     (0, 'assert_expr') ),
+
+            'assert_not_pypy':	( '%|assert not %c\n' , (1, 'assert_exp') ),
+            'assert2_not_pypy':	( '%|assert not %c, %c\n' , (1, 'assert_exp'),
+                                  (4, 'expr') ),
+            'assert2_pypy':	( '%|assert %c, %c\n' , (1, 'assert_expr'),
+                                  (4, 'expr') ),
             'try_except_pypy':	   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
             'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ),
             'assign3_pypy':        ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ),

uncompyle6/semantics/customize26_27.py

@@ -40,6 +40,7 @@ def customize_for_version26_27(self, version):
             'testtrue_then': ( 'not %p', (0, 22) ),
         })
 
+    # FIXME: this should be a transformation
     def n_call(node):
         mapping = self._get_mapping(node)
         key = node

uncompyle6/semantics/customize3.py

@@ -181,6 +181,19 @@ def customize_for_version3(self, version):
         # the iteration variable. These rules we can ignore
         # since we pick up the iteration variable some other way and
         # we definitely don't include in the source  _[dd].
+        TABLE_DIRECT.update({
+            "ifstmt30":	( "%|if %c:\n%+%c%-",
+                          (0, "testfalse_then"),
+                          (1, "_ifstmts_jump30") ),
+            "ifnotstmt30": ( "%|if not %c:\n%+%c%-",
+                             (0, "testtrue_then"),
+                             (1, "_ifstmts_jump30") ),
+            "try_except30": ( "%|try:\n%+%c%-%c\n\n",
+                              (1, "suite_stmts_opt"),
+                              (4, "except_handler") ),
+
+            })
+
         def n_comp_iter(node):
             if node[0] == "expr":
                 n = node[0][0]
@@ -197,11 +210,14 @@ def customize_for_version3(self, version):
         # FIXME: perhaps this can be folded into the 3.4+ case?
         def n_yield_from(node):
             assert node[0] == "expr"
-            assert node[0][0] == "get_iter"
-            # Skip over yield_from.expr.get_iter which adds an
-            # extra iter(). Maybe we can do in tranformation phase instead?
-            template = ("yield from %c", (0, "expr"))
-            self.template_engine(template, node[0][0])
+            if node[0][0] == "get_iter":
+                # Skip over yield_from.expr.get_iter which adds an
+                # extra iter(). Maybe we can do in tranformation phase instead?
+                template = ("yield from %c", (0, "expr"))
+                self.template_engine(template, node[0][0])
+            else:
+                template = ("yield from %c", (0, "attribute"))
+                self.template_engine(template, node[0][0][0])
             self.prune()
 
         self.n_yield_from = n_yield_from

uncompyle6/semantics/customize36.py

@@ -495,7 +495,10 @@ def customize_for_version36(self, version):
                 # bytecode, the escaping of the braces has been
                 # removed. So we need to put back the braces escaping in
                 # reconstructing the source.
-                assert expr[0] == 'LOAD_STR'
+                assert (
+                    expr[0] == "LOAD_STR" or
+                    expr[0] == "LOAD_CONST" and isinstance(expr[0].attr, unicode)
+                    )
                 value = value.replace("{", "{{").replace("}", "}}")
 
             # Remove leading quotes

uncompyle6/semantics/fragments.py

@@ -187,6 +187,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.hide_internal = False
         self.offsets = {}
         self.last_finish = -1
+        self.is_pypy = is_pypy
 
         # FIXME: is there a better way?
         global MAP_DIRECT_FRAGMENT
@@ -789,7 +790,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         self.name = code_name
 
         # Issue created with later Python code generation is that there
-        # is a lamda set up with a dummy argument name that is then called
+        # is a lambda set up with a dummy argument name that is then called
         # So we can't just translate that as is but need to replace the
         # dummy name. Below we are picking out the variable name as seen
         # in the code. And trying to generate code for the other parts
@@ -1452,25 +1453,80 @@ class FragmentsWalker(pysource.SourceWalker, object):
         # as a custom rule
         start = len(self.f.getvalue())
         n = len(node) - 1
-        assert node[n].kind.startswith("CALL_FUNCTION")
 
-        for i in range(n - 2, 0, -1):
-            if not node[i].kind in ["expr", "LOAD_CLASSNAME"]:
-                break
-            pass
+        if node.kind != "expr":
+            if node == "kwarg":
+                self.template_engine(("(%[0]{attr}=%c)", 1), node)
+                return
 
-        if i == n - 2:
-            return
-        self.write("(")
-        line_separator = ", "
-        sep = ""
-        i += 1
-        while i < n:
-            value = self.traverse(node[i])
-            self.node_append(sep, value, node[i])
+            kwargs = None
+            assert node[n].kind.startswith("CALL_FUNCTION")
+
+            if node[n].kind.startswith("CALL_FUNCTION_KW"):
+                if self.is_pypy:
+                    # FIXME: this doesn't handle positional and keyword args
+                    # properly. Need to do something more like that below
+                    # in the non-PYPY 3.6 case.
+                    self.template_engine(('(%[0]{attr}=%c)', 1), node[n-1])
+                    return
+                else:
+                    kwargs = node[n - 1].attr
+
+                assert isinstance(kwargs, tuple)
+                i = n - (len(kwargs) + 1)
+                j = 1 + n - node[n].attr
+            else:
+                i = start = n - 2
+                for i in range(start, 0, -1):
+                    if not node[i].kind in ["expr", "call", "LOAD_CLASSNAME"]:
+                        break
+                    pass
+
+                if i == start:
+                    return
+                i += 2
+
+            for i in range(n - 2, 0, -1):
+                if not node[i].kind in ["expr", "LOAD_CLASSNAME"]:
+                    break
+                pass
+
+            line_separator = ", "
+            sep = ""
             i += 1
-            self.write(sep, value)
-            sep = line_separator
+            self.write("(")
+            if kwargs:
+                # 3.6+ does this
+                while j < i:
+                    self.write(sep)
+                    value = self.traverse(node[j])
+                    self.write("%s" % value)
+                    sep = line_separator
+                    j += 1
+
+                j = 0
+                while i < l:
+                    self.write(sep)
+                    value = self.traverse(node[i])
+                    self.write("%s=%s" % (kwargs[j], value))
+                    sep = line_separator
+                    j += 1
+                    i += 1
+            else:
+                while i < l:
+                    value = self.traverse(node[i])
+                    i += 1
+                    self.write(sep, value)
+                    sep = line_separator
+                    pass
+            pass
+        else:
+            if self.version >= 3.6 and node[0] == "LOAD_CONST":
+                return
+            value = self.traverse(node[0])
+            self.write("(")
+            self.write(value)
+            pass
 
         self.write(")")
         self.set_pos_info(node, start, len(self.f.getvalue()))

uncompyle6/semantics/pysource.py

@@ -239,7 +239,9 @@ class SourceWalker(GenericASTTraversal, object):
             is_pypy=is_pypy,
         )
 
-        self.treeTransform = TreeTransform(version, showast)
+        self.treeTransform = TreeTransform(version=version,
+                                           show_ast=showast,
+                                           is_pypy=is_pypy)
         self.debug_parser = dict(debug_parser)
         self.showast = showast
         self.params = params
@@ -1564,8 +1566,15 @@ class SourceWalker(GenericASTTraversal, object):
             assert node[n].kind.startswith("CALL_FUNCTION")
 
             if node[n].kind.startswith("CALL_FUNCTION_KW"):
-                # 3.6+ starts doing this
-                kwargs = node[n - 1].attr
+                if self.is_pypy:
+                    # FIXME: this doesn't handle positional and keyword args
+                    # properly. Need to do something more like that below
+                    # in the non-PYPY 3.6 case.
+                    self.template_engine(('(%[0]{attr}=%c)', 1), node[n-1])
+                    return
+                else:
+                    kwargs = node[n - 1].attr
+
                 assert isinstance(kwargs, tuple)
                 i = n - (len(kwargs) + 1)
                 j = 1 + n - node[n].attr
@@ -1750,65 +1759,95 @@ class SourceWalker(GenericASTTraversal, object):
                 else:
                     kv_node = node[1:]
             else:
-                assert node[-1].kind.startswith("kvlist")
-                kv_node = node[-1]
-
-            first_time = True
-            for kv in kv_node:
-                assert kv in ("kv", "kv2", "kv3")
-
-                # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
-                # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
-                # kv3 ::= expr expr STORE_MAP
-
-                # FIXME: DRY this and the above
                 indent = self.indent + "  "
-                if kv == "kv":
-                    self.write(sep)
-                    name = self.traverse(kv[-2], indent="")
-                    if first_time:
-                        line_number = self.indent_if_source_nl(line_number, indent)
-                        first_time = False
+                line_number = self.line_number
+                sep = ''
+                opname = node[-1].kind
+                if self.is_pypy and self.version >= 3.5:
+                    if opname.startswith('BUILD_CONST_KEY_MAP'):
+                        keys = node[-2].attr
+                        # FIXME: DRY this and the above
+                        for i in range(len(keys)):
+                            key = keys[i]
+                            value = self.traverse(node[i], indent='')
+                            self.write(sep, key, ': ', value)
+                            sep = ", "
+                            if line_number != self.line_number:
+                                sep += "\n" + self.indent + "  "
+                                line_number = self.line_number
+                                pass
+                            pass
                         pass
-                    line_number = self.line_number
-                    self.write(name, ": ")
-                    value = self.traverse(
-                        kv[1], indent=self.indent + (len(name) + 2) * " "
-                    )
-                elif kv == "kv2":
-                    self.write(sep)
-                    name = self.traverse(kv[1], indent="")
-                    if first_time:
-                        line_number = self.indent_if_source_nl(line_number, indent)
-                        first_time = False
+                    else:
+                        if opname.startswith('kvlist'):
+                            list_node = node[0]
+                        else:
+                            list_node = node
+
+                        assert list_node[-1].kind.startswith('BUILD_MAP')
+                        for i in range(0, len(list_node)-1, 2):
+                            key = self.traverse(list_node[i], indent='')
+                            value = self.traverse(list_node[i+1], indent='')
+                            self.write(sep, key, ': ', value)
+                            sep = ", "
+                            if line_number != self.line_number:
+                                sep += "\n" + self.indent + "  "
+                                line_number = self.line_number
+                                pass
+                            pass
                         pass
-                    line_number = self.line_number
-                    self.write(name, ": ")
-                    value = self.traverse(
-                        kv[-3], indent=self.indent + (len(name) + 2) * " "
-                    )
-                elif kv == "kv3":
-                    self.write(sep)
-                    name = self.traverse(kv[-2], indent="")
-                    if first_time:
-                        line_number = self.indent_if_source_nl(line_number, indent)
-                        first_time = False
+                elif opname.startswith('kvlist'):
+                    kv_node = node[-1]
+                    first_time = True
+                    for kv in kv_node:
+                        assert kv in ('kv', 'kv2', 'kv3')
+
+                        # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR
+                        # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR
+                        # kv3 ::= expr expr STORE_MAP
+
+                        # FIXME: DRY this and the above
+                        if kv == 'kv':
+                            self.write(sep)
+                            name = self.traverse(kv[-2], indent='')
+                            if first_time:
+                                line_number = self.indent_if_source_nl(line_number, indent)
+                                first_time = False
+                                pass
+                            line_number = self.line_number
+                            self.write(name, ': ')
+                            value = self.traverse(kv[1], indent=self.indent+(len(name)+2)*' ')
+                        elif kv == 'kv2':
+                            self.write(sep)
+                            name = self.traverse(kv[1], indent='')
+                            if first_time:
+                                line_number = self.indent_if_source_nl(line_number, indent)
+                                first_time = False
+                                pass
+                            line_number = self.line_number
+                            self.write(name, ': ')
+                            value = self.traverse(kv[-3], indent=self.indent+(len(name)+2)*' ')
+                        elif kv == 'kv3':
+                            self.write(sep)
+                            name = self.traverse(kv[-2], indent='')
+                            if first_time:
+                                line_number = self.indent_if_source_nl(line_number, indent)
+                                first_time = False
+                                pass
+                            line_number = self.line_number
+                            self.write(name, ': ')
+                            line_number = self.line_number
+                            value = self.traverse(kv[0], indent=self.indent+(len(name)+2)*' ')
+                            pass
+                        self.write(value)
+                        sep = ", "
+                        if line_number != self.line_number:
+                            sep += "\n" + self.indent + "  "
+                            line_number = self.line_number
+                            pass
                         pass
-                    line_number = self.line_number
-                    self.write(name, ": ")
-                    line_number = self.line_number
-                    value = self.traverse(
-                        kv[0], indent=self.indent + (len(name) + 2) * " "
-                    )
-                    pass
-                self.write(value)
-                sep = ", "
-                if line_number != self.line_number:
-                    sep += "\n" + self.indent + "  "
-                    line_number = self.line_number
-                    pass
+
                 pass
-            pass
         if sep.startswith(",\n"):
             self.write(sep[1:])
         if node[0] != "dict_entry":

uncompyle6/semantics/transform.py

@@ -30,9 +30,11 @@ def is_docstring(node):
 
 
 class TreeTransform(GenericASTTraversal, object):
-    def __init__(self, version, show_ast=None):
+    def __init__(self, version, show_ast=None,
+                is_pypy=False):
         self.version = version
         self.showast = show_ast
+        self.is_pypy = is_pypy
         return
 
     def maybe_show_tree(self, ast):
@@ -113,11 +115,14 @@ class TreeTransform(GenericASTTraversal, object):
 
                     call = expr[0]
                     LOAD_ASSERT = call[0]
-                    expr = call[1][0]
-                    node = SyntaxTree(
-                        kind,
-                        [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1]
-                    )
+                    if isinstance(call[1], SyntaxTree):
+                        expr = call[1][0]
+                        node = SyntaxTree(
+                            kind,
+                            [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1]
+                        )
+                        pass
+                    pass
                 else:
                     # ifstmt
                     #   0. testexpr (2)
@@ -133,7 +138,10 @@ class TreeTransform(GenericASTTraversal, object):
                     # becomes:
                     # assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
                     if jump_cond == "jmp_true":
-                        kind = "assert"
+                        if self.is_pypy:
+                            kind = "assert0_pypy"
+                        else:
+                            kind = "assert"
                     else:
                         assert jump_cond == "jmp_false"
                         kind = "assertnot"
@@ -230,6 +238,15 @@ class TreeTransform(GenericASTTraversal, object):
 
     n_ifelsestmtc = n_ifelsestmtl = n_ifelsestmt
 
+    def n_list_for(self, list_for_node):
+        expr = list_for_node[0]
+        if (expr == "expr" and expr[0] == "get_iter"):
+            # Remove extraneous get_iter() inside the "for" of a comprehension
+            assert expr[0][0] == "expr"
+            list_for_node[0] = expr[0][0]
+            list_for_node.transformed_by="n_list_for",
+        return list_for_node
+
     def traverse(self, node, is_lambda=False):
         node = self.preorder(node)
         return node

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION="3.5.0"  # noqa
+VERSION="3.5.1"  # noqa