Merge branch 'python-3.3-to-3.5' into python-3.0-to-3.2

.github/FUNDING.yml

@@ -6,7 +6,7 @@ open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
+liberapay: rocky
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
 custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

test/stdlib/3.2-exclude.sh

@@ -1,7 +1,14 @@
 SKIP_TESTS=(
-    [test_descr.py]=1 # FIXME: Works on c90ff51?
+    [test_descr.py]=1
+    # [test_descr.py]=pytest_module # FIXME: Works on c90ff51?
+    # AssertionError: 'D(4)C(4)A(4)' != 'D(4)C(4)B(4)A(4)'
+    # - D(4)C(4)A(4)
+    # + D(4)C(4)B(4)A(4)
+    # ?         ++++
 
-    [test_cmath.py]=1 # FIXME
+
+    [test_cmath.py]=1 # Control-flow "elif else -> else: if else"
+    # [test_cmath.py]=pytest_module
     # AssertionError: rect1000: rect(complex(0.0, 0.0))
     # Expected: complex(0.0, 0.0)
     # Received: complex(0.0, -1.0)
@@ -9,14 +16,27 @@ SKIP_TESTS=(
 
 
     [test_cmd_line.py]=1
-    [test_collections.py]=1
+
+    [test_collections.py]=1 # fail on its own
+    # E   TypeError: __new__() takes exactly 4 arguments (1 given)
+
     [test_concurrent_futures.py]=1 # too long to run over 46 seconds by itself
-    [test_datetimetester.py]=1
+    [test_datetime.py]=pytest_module
-    [test_decimal.py]=1
+
-    [test_dictcomps.py]=1 # FIXME: semantic error: actual = {k:v for k in }
+    [test_decimal.py]=1 # Fails on its own, even with pytest
-    [test_doctest.py]=1 # test failures
+
+    [test_dictcomps.py]=1
+    # [test_dictcomps.py]=pytest_module # FIXME: semantic error: actual = {k:v for k in }
+    #    assert (count * 2) <= i
+
+    [test_doctest.py]=1 # Missing pytest fixture
+    # [test_doctest.py]=pytest_module
+    # fixture 'coverdir' not found
+
     [test_dis.py]=1   # We change line numbers - duh!
+
     [test_exceptions.py]=1 # parse error
+    # [test_exceptions.py]=pytest_module # parse error
 
     [test_modulefinder.py]=1 # test failures
     [test_multiprocessing.py]=1 # test takes too long to run: 35 seconds

test/stdlib/3.3-exclude.sh

@@ -10,7 +10,7 @@ SKIP_TESTS=(
     #       tgt.append(elem)
     [test_itertools.py]=1
 
-    [test_buffer.py]=1  # FIXME: Works on c90ff51
+    [test_buffer.py]=pytest  # FIXME: Works on c90ff51
     [test_cmath.py]=pytest
 
     [test_atexit.py]=1  # The atexit test starting at 3.3 looks for specific comments in error lines

test/stdlib/3.8-exclude.sh

@@ -28,12 +28,12 @@ SKIP_TESTS=(
 
     # These and the above may be due to new code generation or tests
     # between 3.8.3 and 3.8.5 ?
-    [test_decorators.py]=1 #
+    [test_decorators.py]=1 # parse error
 
-    [test_dtrace.py]=1 #
+    [test_dtrace.py]=1 # parse error
-    [test_exceptions.py]=1 #
+    [test_exceptions.py]=1 # parse error
     [test_ftplib.py]=1 #
-    [test_gc.py]=1 #
+    [test_gc.py]=1 # FIXME: return                 return strip_python_stderr(stderr)
     [test_gzip.py]=1 #
     [test_hashlib.py]=1 #
     [test_iter.py]=1 #
@@ -51,7 +51,6 @@ SKIP_TESTS=(
     [test_audioop.py]=1  # test failure
     [test_audit.py]=1  # parse error
 
-    [test_base64.py]=1  # parse error
     [test_baseexception.py]=1  #
     [test_bigaddrspace.py]=1  # parse error
     [test_bigmem.py]=1  # parse error

test/stdlib/runtests.sh

@@ -168,12 +168,8 @@ if ((IS_PYPY)); then
 else
     cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
 fi
-if [[ $PYVERSION == 3.2 ]] ; then
-    cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR
-    cd $TESTDIR
-else
 cd $TESTDIR/test
-fi
+
 pyenv local $FULLVERSION
 export PYTHONPATH=$TESTDIR
 export PATH=${PYENV_ROOT}/shims:${PATH}
@@ -188,7 +184,7 @@ if [[ -n $1 ]] ; then
     typeset -a files_ary=( $(echo $@) )
     if (( ${#files_ary[@]} == 1 || DONT_SKIP_TESTS == 1 )) ; then
 	for file in $files; do
-	    if (( SKIP_TESTS[$file] != "pytest" )); then
+	    if (( SKIP_TESTS[$file] != "pytest" || SKIP_TESTS[$file] != "pytest_module" )); then
 	       SKIP_TESTS[$file]=1;
 	    fi
 	done
@@ -208,6 +204,8 @@ for file in $files; do
 
     if [[ ${SKIP_TESTS[$file]} == "pytest" ]]; then
 	PYTHON=pytest
+    elif [[ ${SKIP_TESTS[$file]} == "pytest_module" ]]; then
+	PYTHON="$PYTHON -m pytest"
     else
 	if [[ ${SKIP_TESTS[$file]}s == ${NOT_INVERTED_TESTS} ]] ; then
 	    ((skipped++))

uncompyle6/parsers/parse2.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2021 Rocky Bernstein
+#  Copyright (c) 2015-2021, 2024 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
 #  Copyright (c) 1999 John Aycock
@@ -27,11 +27,12 @@ that a later phase can turn into a sequence of ASCII text.
 
 from __future__ import print_function
 
-from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
-from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
-from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
+from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
+from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
+from uncompyle6.parsers.treenode import SyntaxTree
+
 
 class Python2Parser(PythonParser):
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -405,7 +406,6 @@ class Python2Parser(PythonParser):
                 "CALL_FUNCTION_VAR_KW",
                 "CALL_FUNCTION_KW",
             ):
-
                 args_pos, args_kw = self.get_pos_kw(token)
 
                 # number of apply equiv arguments:
@@ -526,7 +526,7 @@ class Python2Parser(PythonParser):
                 custom_seen_ops.add(opname)
                 continue
             elif opname == "LOAD_LISTCOMP":
-                self.addRule("expr ::= listcomp", nop_func)
+                self.addRule("expr ::= list_comp", nop_func)
                 custom_seen_ops.add(opname)
                 continue
             elif opname == "LOAD_SETCOMP":

uncompyle6/parsers/parse3.py

@@ -1085,7 +1085,9 @@ class Python3Parser(PythonParser):
                 )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_LISTCOMP":
-                self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_SETCOMP":
                 # Should this be generalized and put under MAKE_FUNCTION?
@@ -1154,7 +1156,7 @@ class Python3Parser(PythonParser):
                             #   and have GET_ITER CALL_FUNCTION_1
                             # Todo: For Pypy we need to modify this slightly
                             rule_pat = (
-                                "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
+                                "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
                                 % ("pos_arg " * pos_args_count, opname)
                             )
@@ -1348,14 +1350,14 @@ class Python3Parser(PythonParser):
                                 # 'exprs' in the rule above into a
                                 # tuple.
                                 rule_pat = (
-                                    "listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
+                                    "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
                                     "expr GET_ITER CALL_FUNCTION_1" % (opname,)
                                 )
                                 self.add_make_function_rule(
                                     rule_pat, opname, token.attr, customize
                                 )
                             rule_pat = (
-                                "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                                "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
                                 % ("expr " * pos_args_count, opname)
                             )
@@ -1399,7 +1401,7 @@ class Python3Parser(PythonParser):
                         #   and have GET_ITER CALL_FUNCTION_1
                         # Todo: For Pypy we need to modify this slightly
                         rule_pat = (
-                            "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1"
                             % ("expr " * pos_args_count, opname)
                         )

uncompyle6/parsers/parse37base.py

@@ -723,7 +723,9 @@ class Python37BaseParser(PythonParser):
                 )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_LISTCOMP":
-                self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_NAME":
                 if (
@@ -802,7 +804,7 @@ class Python37BaseParser(PythonParser):
                             #   and have GET_ITER CALL_FUNCTION_1
                             # Todo: For Pypy we need to modify this slightly
                             rule_pat = (
-                                "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
+                                "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
                                 % ("pos_arg " * args_pos, opname)
                             )
@@ -900,14 +902,14 @@ class Python37BaseParser(PythonParser):
                         # 'exprs' in the rule above into a
                         # tuple.
                         rule_pat = (
-                            "listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
+                            "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
                             "expr GET_ITER CALL_FUNCTION_1" % (opname,)
                         )
                         self.add_make_function_rule(
                             rule_pat, opname, token.attr, customize
                         )
                         rule_pat = (
-                            "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
                         )
                         self.add_make_function_rule(
@@ -941,7 +943,7 @@ class Python37BaseParser(PythonParser):
                         #   and have GET_ITER CALL_FUNCTION_1
                         # Todo: For Pypy we need to modify this slightly
                         rule_pat = (
-                            "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
                         )
                         self.add_make_function_rule(

uncompyle6/parsers/parse38.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2020, 2022-2023 Rocky Bernstein
+#  Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,9 +17,12 @@ spark grammar differences over Python 3.7 for Python 3.8
 """
 from __future__ import print_function
 
-from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse37 import Python37Parser
+from uncompyle6.parsers.reducecheck.pop_return import pop_return_check
+
 
 class Python38Parser(Python37Parser):
     def p_38_stmt(self, args):
@@ -362,13 +365,24 @@ class Python38Parser(Python37Parser):
         """
         )
 
-    def customize_grammar_rules(self, tokens, customize):
+    def customize_reduce_checks_full38(self, tokens, customize):
-        super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+        """
+        Extra tests when a reduction is made in the full grammar.
+
+        Reductions here are extended from those used in the lambda grammar
+        """
         self.remove_rules_38()
+        self.check_reduce["pop_return"] = "tokens"
         self.check_reduce["whileTruestmt38"] = "tokens"
         self.check_reduce["whilestmt38"] = "tokens"
         self.check_reduce["try_elsestmtl38"] = "AST"
 
+        self.reduce_check_table["pop_return"] = pop_return_check
+
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+        self.customize_reduce_checks_full38(tokens, customize)
+
         # For a rough break out on the first word. This may
         # include instructions that don't need customization,
         # but we'll do a finer check after the rough breakout.
@@ -423,11 +437,7 @@ class Python38Parser(Python37Parser):
         # Determine if we have an iteration CALL_FUNCTION_1.
         has_get_iter_call_function1 = False
         for i, token in enumerate(tokens):
-            if (
+            if token == "GET_ITER" and i < n - 2 and tokens[i + 1] == "CALL_FUNCTION_1":
-                token == "GET_ITER"
-                and i < n - 2
-                and tokens[i + 1] == "CALL_FUNCTION_1"
-            ):
                 has_get_iter_call_function1 = True
 
         for i, token in enumerate(tokens):
@@ -577,14 +587,10 @@ class Python38Parser(Python37Parser):
                        """
                     self.addRule(rule, nop_func)
 
-
-
-
-
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
-        invalid = super(Python38Parser,
+        invalid = super(Python38Parser, self).reduce_is_invalid(
-                        self).reduce_is_invalid(rule, ast,
+            rule, ast, tokens, first, last
-                                                tokens, first, last)
+        )
         self.remove_rules_38()
         if invalid:
             return invalid
@@ -612,7 +618,7 @@ if __name__ == "__main__":
     p = Python38Parser()
     p.remove_rules_38()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
 
     if PYTHON_VERSION_TRIPLE[:2] == (3, 8):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
@@ -635,7 +641,9 @@ if __name__ == "__main__":
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         import sys
+
         if len(sys.argv) > 1:
             from spark_parser.spark import rule2str
+
             for rule in sorted(p.rule2name.items()):
                 print(rule2str(rule[0]))

uncompyle6/parsers/reducecheck/pop_return.py

@@ -0,0 +1,10 @@
+#  Copyright (c) 2020 Rocky Bernstein
+
+
+def pop_return_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # If the first instruction of return_expr (the instruction after POP_TOP) is
+    # has a linestart, then the POP_TOP was probably part of the previous
+    # statement, such as a call() where the return value is discarded.
+    return tokens[first + 1].linestart

uncompyle6/scanner.py

@@ -89,7 +89,7 @@ def long(num):
 CONST_COLLECTIONS = ("CONST_LIST", "CONST_SET", "CONST_DICT", "CONST_MAP")
 
 
-class Code(object):
+class Code:
     """
     Class for representing code-objects.
 
@@ -292,7 +292,13 @@ class Scanner:
             return False
         return offset < self.get_target(offset)
 
-    def prev_offset(self, offset):
+    def ingest(self, co, classname=None, code_objects={}, show_asm=None):
+        """
+        Code to tokenize disassembly. Subclasses must implement this.
+        """
+        raise NotImplementedError("This method should have been implemented")
+
+    def prev_offset(self, offset: int) -> int:
         return self.insts[self.offset2inst_index[offset] - 1].offset
 
     def get_inst(self, offset):

uncompyle6/scanners/scanner37.py

@@ -52,7 +52,7 @@ class Scanner37(Scanner37Base):
         if collection_type == "CONST_DICT":
             # constant dictionaries work via BUILD_CONST_KEY_MAP and
             # handle the values() like sets and lists.
-            # However the keys() are an LOAD_CONST of the keys.
+            # However, the keys() are an LOAD_CONST of the keys.
             # adjust offset to account for this
             count += 1
 

uncompyle6/scanners/scanner37base.py

@@ -263,10 +263,9 @@ class Scanner37Base(Scanner):
                 if (
                     next_inst.opname == "LOAD_GLOBAL"
                     and next_inst.argval == "AssertionError"
-                    and inst.argval
+                    and inst.argval is not None
                 ):
-                    raise_idx = self.offset2inst_index[self.prev_op[inst.argval]]
+                    raise_inst = self.get_inst(self.prev_op[inst.argval])
-                    raise_inst = self.insts[raise_idx]
                     if raise_inst.opname.startswith("RAISE_VARARGS"):
                         self.load_asserts.add(next_inst.offset)
                     pass
@@ -283,7 +282,7 @@ class Scanner37Base(Scanner):
             # some backward jumps, are turned into forward jumps to another
             # "extended arg" backward jump to the same location.
             if inst.opname == "JUMP_FORWARD":
-                jump_inst = self.insts[self.offset2inst_index[inst.argval]]
+                jump_inst = self.get_inst(inst.argval)
                 if jump_inst.has_extended_arg and jump_inst.opname.startswith("JUMP"):
                     # Create a combination of the jump-to instruction and
                     # this one. Keep the position information of this instruction,

uncompyle6/semantics/gencomp.py

@@ -21,9 +21,9 @@ from xdis import co_flags_is_async, iscode
 
 from uncompyle6.parser import get_python_parser
 from uncompyle6.scanner import Code
+from uncompyle6.scanners.tok import Token
 from uncompyle6.semantics.consts import PRECEDENCE
 from uncompyle6.semantics.helper import is_lambda_mode
-from uncompyle6.scanners.tok import Token
 
 
 class ComprehensionMixin:
@@ -172,7 +172,10 @@ class ComprehensionMixin:
                 tree = tree[1]
             pass
 
-        if tree in ("genexpr_func", "genexpr_func_async",):
+        if tree in (
+            "genexpr_func",
+            "genexpr_func_async",
+        ):
             for i in range(3, 5):
                 if tree[i] == "comp_iter":
                     iter_index = i
@@ -330,8 +333,19 @@ class ComprehensionMixin:
             assert store == "store"
             n = set_iter_async[2]
         elif node == "list_comp" and tree[0] == "expr":
+            list_iter = None
+            for list_iter_try in tree:
+                if list_iter_try == "list_iter":
+                    list_iter = list_iter_try
+                    break
+                if not list_iter_try:
                     tree = tree[0][0]
                     n = tree[iter_index]
+                else:
+                    n = list_iter
+                    pass
+                pass
+            pass
         else:
             n = tree[iter_index]
 
@@ -405,6 +419,9 @@ class ComprehensionMixin:
                 n = n[0]
 
             if n in ("list_for", "comp_for"):
+                if n == "list_for" and not comp_for and n[0] == "expr":
+                    comp_for = n[0]
+
                 n_index = 3
                 if (
                     (n[2] == "store")
@@ -494,11 +511,21 @@ class ComprehensionMixin:
         if comp_for:
             self.preorder(comp_for)
         else:
+            try:
+                node[in_node_index]
+            except:
+                from trepan.api import debug
+
+                debug()
             self.preorder(node[in_node_index])
 
         # Here is where we handle nested list iterations.
         if tree == "list_comp" and self.version != (3, 0):
-            list_iter = tree[1]
+            list_iter = None
+            for list_iter_try in tree:
+                if list_iter_try == "list_iter":
+                    list_iter = list_iter_try
+                    break
             assert list_iter == "list_iter"
             if list_iter[0] == "list_for":
                 self.preorder(list_iter[0][3])
@@ -637,7 +664,6 @@ class ComprehensionMixin:
             # Find the list comprehension body. It is the inner-most
             # node that is not list_.. .
             while n == "list_iter":
-
                 # recurse one step
                 n = n[0]
 

uncompyle6/semantics/n_actions.py

@@ -1036,7 +1036,7 @@ class NonterminalActions:
         self.prec = p
         self.prune()  # stop recursing
 
-    def n_listcomp(self, node):
+    def n_list_comp(self, node):
         self.write("[")
         if node[0].kind == "load_closure":
             assert self.version >= (3, 0)