Merge branch 'master' into python-3.3-to-3.5

admin-tools/pyenv-newest-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.13 pypy3.8-7.3.9 pyston-2.3.3 3.8.13'
+export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.14 pypy3.8-7.3.9 pyston-2.3.3 3.8.14'

test/simple_source/bug36/10_fstring.py

@@ -59,9 +59,9 @@ log_rounds = 5
 assert "05$" == f"{log_rounds:02d}$"
 
 
-def testit(a, b, l):
+def testit(a, b, ll):
-    # print(l)
+    # print(ll)
-    return l
+    return ll
 
 
 # The call below shows the need for BUILD_STRING to count expr arguments.
@@ -102,11 +102,11 @@ assert f"x={x*y:{width}}" == "x=foofoo    "
 # equivalent thing. For compatiblity with older Python we'll use "%"
 # instead of a format string
 def f():
-    f"""Not a docstring"""
+    f"""Not a docstring"""  # noqa
 
 
 def g():
-    """Not a docstring""" f""
+    """Not a docstring""" f""  # noqa
 
 
 assert f.__doc__ is None
@@ -129,3 +129,17 @@ assert f'{f"{0}"*3}' == "000"
 # The former, {{ confuses the format strings so dictionary/set comprehensions
 # don't work.
 assert f"expr={ {x: y for x, y in [(1, 2), ]}}" == "expr={1: 2}"
+
+
+class Line:
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+
+    # From 3.7 test_typing.py
+    def __str__(self):
+        return f"{self.x} -> {self.y}"
+
+
+line = Line(1, 2)
+assert str(line) == "1 -> 2"

test/stdlib/runtests.sh

@@ -29,7 +29,19 @@ function displaytime {
 # Python version setup
 FULLVERSION=$(pyenv local)
 PYVERSION=${FULLVERSION%.*}
-MINOR=${FULLVERSION##?.?.}
+
+if [[ $PYVERSION =~ 'pypy' ]] ; then
+    IS_PYPY=1
+else
+    IS_PYPY=0
+fi
+
+if [[ $FULLVERSION =~ pypy([2-3])\.([7-9]) ]]; then
+    MAJOR="${BASH_REMATCH[1]}"
+    MINOR="${BASH_REMATCH[2]}"
+else
+   MINOR=${FULLVERSION##?.?.}
+fi
 
 STOP_ONERROR=${STOP_ONERROR:-1}
 
@@ -150,7 +162,12 @@ done
 
 
 mkdir $TESTDIR || exit $?
-cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
+
+if ((IS_PYPY)); then
+    cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib-python/${MAJOR}/test $TESTDIR
+else
+    cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
+fi
 if [[ $PYVERSION == 3.2 ]] ; then
     cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR
     cd $TESTDIR
@@ -209,7 +226,11 @@ for file in $files; do
     ((i++))
     # (( i > 40 )) && break
     short_name=$(basename $file .py)
-    decompiled_file=$short_name-${PYVERSION}.pyc
+    if ((IS_PYPY)); then
+	decompiled_file=$short_name-${MAJOR}.${MINOR}.pyc
+    else
+	decompiled_file=$short_name-${PYVERSION}.pyc
+    fi
     $fulldir/compile-file.py $file && \
     mv $file{,.orig} && \
     echo ==========  $(date +%X) Decompiling $file ===========

uncompyle6/parsers/parse37.py

@@ -138,7 +138,8 @@ class Python37Parser(Python37BaseParser):
         returns ::= _stmts return
 
         stmt ::= genexpr_func
-        genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter JUMP_BACK
+        genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter
+                         _come_froms JUMP_BACK _come_froms
         """
         pass
 
@@ -1365,7 +1366,7 @@ class Python37Parser(Python37BaseParser):
 
                         genexpr_func_async  ::= LOAD_ARG func_async_prefix
                                                 store func_async_middle comp_iter
-                                                JUMP_LOOP COME_FROM
+                                                JUMP_BACK COME_FROM
                                                 POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
 
                         # FIXME this is a workaround for probalby some bug in the Earley parser
@@ -1377,7 +1378,7 @@ class Python37Parser(Python37BaseParser):
 
                         list_afor2          ::= func_async_prefix
                                                 store func_async_middle list_iter
-                                                JUMP_LOOP COME_FROM
+                                                JUMP_BACK COME_FROM
                                                 POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
 
                         list_comp_async     ::= BUILD_LIST_0 LOAD_ARG list_afor2
@@ -1463,13 +1464,13 @@ class Python37Parser(Python37BaseParser):
                     genexpr_func_async   ::= LOAD_ARG async_iter
                                              store_async_iter_end
                                              comp_iter
-                                             JUMP_LOOP COME_FROM
+                                             JUMP_BACK COME_FROM
                                              POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
 
                     list_afor2           ::= async_iter
                                              store
                                              list_iter
-                                             JUMP_LOOP
+                                             JUMP_BACK
                                              COME_FROM_FINALLY
                                              END_ASYNC_FOR
 
@@ -1479,7 +1480,7 @@ class Python37Parser(Python37BaseParser):
                                              store
                                              func_async_middle
                                              set_iter
-                                             JUMP_LOOP COME_FROM
+                                             JUMP_BACK COME_FROM
                                              POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
 
                     set_afor2            ::= expr_or_arg
@@ -1490,7 +1491,7 @@ class Python37Parser(Python37BaseParser):
                     set_iter_async       ::= async_iter
                                              store
                                              set_iter
-                                             JUMP_LOOP
+                                             JUMP_BACK
                                              _come_froms
                                              END_ASYNC_FOR
 

uncompyle6/scanners/scanner2.py

@@ -418,7 +418,16 @@ class Scanner2(Scanner):
                     #   either to a FOR_ITER or the instruction after a SETUP_LOOP
                     #   and it is followed by another JUMP_FORWARD
                     # then we'll take it as a "continue".
-                    j = self.offset2inst_index[offset]
+                    j = self.offset2inst_index.get(offset)
+
+                    # EXTENDED_ARG doesn't appear in instructions,
+                    # but is instead the next opcode folded into it, and has the offset
+                    # of the EXTENDED_ARG. Therefor in self.offset2nist_index we'll find
+                    # the instruction at the previous EXTENDED_ARG offset which is 3
+                    # bytes back.
+                    if j is None and offset > self.opc.ARG_MAX_VALUE:
+                        j = self.offset2inst_index[offset - 3]
+
                     target_index = self.offset2inst_index[target]
                     is_continue = (
                         self.insts[target_index - 1].opname == "SETUP_LOOP"

uncompyle6/semantics/customize.py

@@ -17,7 +17,13 @@
 """
 
 from uncompyle6.parsers.treenode import SyntaxTree
-from uncompyle6.semantics.consts import INDENT_PER_LEVEL, NO_PARENTHESIS_EVER, PRECEDENCE, TABLE_R, TABLE_DIRECT
+from uncompyle6.semantics.consts import (
+    INDENT_PER_LEVEL,
+    NO_PARENTHESIS_EVER,
+    PRECEDENCE,
+    TABLE_R,
+    TABLE_DIRECT,
+)
 from uncompyle6.semantics.helper import flatten_list
 from uncompyle6.scanners.tok import Token
 
@@ -27,22 +33,29 @@ def customize_for_version(self, is_pypy, version):
         ########################
         # PyPy changes
         #######################
+        # fmt: off
         TABLE_DIRECT.update({
-            "assert": ("%|assert %c\n", 0),
-            "assert_pypy":	( '%|assert %c\n' ,     (1, 'assert_expr') ),
-            # This is as a result of an if transformation
-            'assert0_pypy':	( '%|assert %c\n' ,     0),
 
-            'assert_not_pypy':	( '%|assert not %c\n' , (1, 'assert_exp') ),
+            "assert":           ("%|assert %c\n", 0),
-            'assert2_not_pypy':	( '%|assert not %c, %c\n' , (1, 'assert_exp'),
+            # This can happen as a result of an if transformation
-                                  (4, 'expr') ),
+            "assert2":          ("%|assert %c, %c\n", 0, 3),
-            'assert2_pypy':	( '%|assert %c, %c\n' , (1, 'assert_expr'),
+            "assert_pypy":	    ( "%|assert %c\n" ,     (1, "assert_expr") ),
-                                  (4, 'expr') ),
+            # This is as a result of an if transformation
-            'try_except_pypy':	   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
+            'assert0_pypy':	    ( "%|assert %c\n" ,     0),
-            'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ),
+
-            'assign3_pypy':        ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ),
+            'assert_not_pypy':	( "%|assert not %c\n" , (1, "assert_exp") ),
-            'assign2_pypy':        ( '%|%c, %c = %c, %c\n', 3, 2, 0, 1),
+            "assert2_not_pypy": (
+                    "%|assert not %c, %c\n",
+                    (1, "assert_exp"),
+                    (4, "expr"),
+                ),
+
+            "try_except_pypy":	   ( "%|try:\n%+%c%-%c\n\n", 1, 2 ),
+            "tryfinallystmt_pypy": ( "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", 1, 3 ),
+            "assign3_pypy":        ( "%|%c, %c, %c = %c, %c, %c\n", 5, 4, 3, 0, 1, 2 ),
+            "assign2_pypy":        ( "%|%c, %c = %c, %c\n", 3, 2, 0, 1),
             })
+        # fmt: on
 
         if version[:2] >= (3, 7):
 
@@ -53,12 +66,18 @@ def customize_for_version(self, is_pypy, version):
                 kw_names = node[-2]
                 assert kw_names == "pypy_kw_keys"
 
-                flat_elems = flatten_list(node[1:-2])
-                n = len(flat_elems)
-                assert n == arg_count
                 kwargs_names = kw_names[0].attr
                 kwarg_count = len(kwargs_names)
                 pos_argc = arg_count - kwarg_count
+
+                flat_elems = flatten_list(node[1:-2])
+                n = len(flat_elems)
+                assert n == arg_count, "n: %s, arg_count: %s\n%s" % (
+                    n,
+                    arg_count,
+                    node,
+                )
+
                 sep = ""
 
                 for i in range(pos_argc):
@@ -96,20 +115,23 @@ def customize_for_version(self, is_pypy, version):
         ########################
         # Without PyPy
         #######################
-        TABLE_DIRECT.update({
+        TABLE_DIRECT.update(
-            # "assert" and "assert_expr" are added via transform rules.
+            {
-            "assert": ("%|assert %c\n", 0),
+                # "assert" and "assert_expr" are added via transform rules.
-            "assert2": ("%|assert %c, %c\n", 0, 3),
+                "assert": ("%|assert %c\n", 0),
-
+                "assert2": ("%|assert %c, %c\n", 0, 3),
-            # Created only via transformation
+                # Created only via transformation
-            "assertnot": ("%|assert not %p\n", (0, PRECEDENCE['unary_not'])),
+                "assertnot": ("%|assert not %p\n", (0, PRECEDENCE["unary_not"])),
-            "assert2not": ( "%|assert not %p, %c\n" ,
+                "assert2not": (
-                            (0, PRECEDENCE['unary_not']), 3 ),
+                    "%|assert not %p, %c\n",
-
+                    (0, PRECEDENCE["unary_not"]),
-            "assign2": ("%|%c, %c = %c, %c\n", 3, 4, 0, 1),
+                    3,
-            "assign3": ("%|%c, %c, %c = %c, %c, %c\n", 5, 6, 7, 0, 1, 2),
+                ),
-            "try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3),
+                "assign2": ("%|%c, %c = %c, %c\n", 3, 4, 0, 1),
-        })
+                "assign3": ("%|%c, %c, %c = %c, %c, %c\n", 5, 6, 7, 0, 1, 2),
+                "try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3),
+            }
+        )
     if version >= (3, 0):
         if version >= (3, 2):
             TABLE_DIRECT.update(
@@ -188,6 +210,7 @@ def customize_for_version(self, is_pypy, version):
                 }
             )
             if version == (2, 4):
+
                 def n_iftrue_stmt24(node):
                     self.template_engine(("%c", 0), node)
                     self.default(node)
@@ -196,6 +219,7 @@ def customize_for_version(self, is_pypy, version):
                 self.n_iftrue_stmt24 = n_iftrue_stmt24
             elif version < (1, 4):
                 from uncompyle6.semantics.customize14 import customize_for_version14
+
                 customize_for_version14(self, version)
 
                 def n_call(node):
@@ -211,9 +235,16 @@ def customize_for_version(self, is_pypy, version):
                             sep = ", "
                         self.write(")")
                     else:
-                        self.template_engine(("%p(%P)",
+                        self.template_engine(
-                                              (0, "expr", 100), (1,-1,", ", NO_PARENTHESIS_EVER)), node)
+                            (
+                                "%p(%P)",
+                                (0, "expr", 100),
+                                (1, -1, ", ", NO_PARENTHESIS_EVER),
+                            ),
+                            node,
+                        )
                     self.prune()
+
                 self.n_call = n_call
 
             else:  # 1.0 <= version <= 2.3:

uncompyle6/semantics/customize36.py

@@ -167,6 +167,8 @@ def customize_for_version36(self, version):
         if node == "classdefdeco2":
             if isinstance(node[1][1].attr, str):
                 class_name = node[1][1].attr
+                if self.is_pypy and class_name.find("<locals>") > 0:
+                    class_name = class_name.split(".")[-1]
             else:
                 class_name = node[1][2].attr
             build_class = node

uncompyle6/semantics/customize37.py

@@ -94,8 +94,7 @@ def customize_for_version37(self, version):
             # need parenthesis.
             # Note there are async dictionary expressions are like await expr's
             # the below is just the default fersion
-            "await_expr": ("await %p", (0, PRECEDENCE["await_expr"]-1)),
+            "await_expr": ("await %p", (0, PRECEDENCE["await_expr"] - 1)),
-
             "await_stmt": ("%|%c\n", 0),
             "c_async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 3),
             "call_ex": ("%c(%p)", (0, "expr"), (1, 100)),
@@ -156,7 +155,10 @@ def customize_for_version37(self, version):
                 (2, "import_from_attr37"),
                 (3, "store"),
             ),
-            "import_one": ("%c", (0, "importlists"),),
+            "import_one": (
+                "%c",
+                (0, "importlists"),
+            ),
             "importattr37": ("%c", (0, "IMPORT_NAME_ATTR")),
             "import_from_attr37": (
                 "%c import %c",
@@ -165,12 +167,9 @@ def customize_for_version37(self, version):
             ),
             "list_afor": (
                 " async for %[1]{%c} in %c%[1]{%c}",
-                (1, "store"), (0, "get_aiter"), (3, "list_iter"),
+                (1, "store"),
-            ),
+                (0, "get_aiter"),
-
+                (3, "list_iter"),
-            "list_afor": (
-                " async for %[1]{%c} in %c%[1]{%c}",
-                (1, "store"), (0, "get_aiter"), (3, "list_iter"),
             ),
 
             "list_if37": (" if %p%c", (0, 27), 1),
@@ -408,12 +407,17 @@ def customize_for_version37(self, version):
     self.n_call = n_call
 
     def n_compare_chained(node):
-        if node[0] == "compare_chained37":
+        if node[0] in (
+            "c_compare_chained37",
+            "c_compare_chained37_false",
+            "compare_chained37",
+            "compare_chained37_false",
+        ):
             self.default(node[0])
         else:
             self.default(node)
 
-    self.n_compare_chained = n_compare_chained
+    self.n_compare_chained = self.n_c_compare_chained = n_compare_chained
 
     def n_importlist37(node):
         if len(node) == 1:
@@ -439,3 +443,26 @@ def customize_for_version37(self, version):
         self.prune()
 
     self.n_list_comp_async = n_list_comp_async
+
+    # FIXME: The following adjusts I guess a bug in the parser.
+    # It might be as simple as renaming grammar symbol "testtrue" to "testtrue_or_false"
+    # and then keeping this as is with the name change.
+    # Fixing in the parsing by inspection is harder than doing it here.
+    def n_testtrue(node):
+        compare_chained37 = node[0]
+        if (
+            compare_chained37 == "compare_chained37"
+            and compare_chained37[1] == "compare_chained1b_37"
+        ):
+            compare_chained1b_37 = compare_chained37[1]
+            if (
+                len(compare_chained1b_37) > 2
+                and compare_chained1b_37[-2] == "JUMP_FORWARD"
+            ):
+                node.kind = "testfalse"
+                pass
+            pass
+        self.default(node)
+        return
+
+    self.n_testtrue = n_testtrue

uncompyle6/semantics/gencomp.py

@@ -99,7 +99,7 @@ class ComprehensionMixin:
         self,
         node,
         iter_index,
-        code_index: int = -5,
+        code_index = -5,
     ):
         p = self.prec
         self.prec = PRECEDENCE["lambda_body"] - 1
@@ -159,6 +159,22 @@ class ComprehensionMixin:
         while len(tree) == 1:
             tree = tree[0]
 
+        if tree == "stmts":
+            # FIXME: rest is a return None?
+            # Verify this
+            # rest = tree[1:]
+            tree = tree[0]
+        elif tree == "lambda_start":
+            assert len(tree) <= 3
+            tree = tree[-2]
+            if tree == "return_expr_lambda":
+                tree = tree[1]
+            pass
+
+        if tree in ("genexpr_func_async",):
+            if tree[3] == "comp_iter":
+                iter_index = 3
+
         n = tree[iter_index]
 
         assert n == "comp_iter", n.kind