Merge branch 'python-3.3-to-3.5' into python-3.0-to-3.2

uncompyle6/scanners/scanner3.py

@@ -297,6 +297,7 @@ class Scanner3(Scanner):
         )
         return new_tokens
 
+    # Move to scanner35?
     def bound_map_from_inst_35(
         self, insts: list, next_tokens: list, t: Token, i: int
     ):
@@ -388,92 +389,6 @@ class Scanner3(Scanner):
         )
         return new_tokens
 
-    def bound_map_from_inst_pre35(
-        self, insts: list, next_tokens: list, t: Token, i: int
-    ):
-        """
-        Try to a sequence of instruction that ends with a BUILD_MAP into
-        a sequence that can be parsed much faster, but inserting the
-        token boundary at the beginning of the sequence.
-        """
-        count = t.attr
-        assert isinstance(count, int)
-
-        # For small lists don't bother
-        if count < 10:
-            return None
-
-        # Older Python BUILD_MAP argument's count is a
-        # key and value pair and STORE_MAP. So it is multiplied by three.
-        collection_end = i + 1 + count * 3
-
-        for j in range(i + 1, collection_end, 3):
-            if insts[j].opname not in ("LOAD_CONST",):
-                return None
-            if insts[j + 1].opname not in ("LOAD_CONST",):
-                return None
-            if insts[j + 2].opname not in ("STORE_MAP",):
-                return None
-
-        collection_enum = CONST_COLLECTIONS.index("CONST_MAP")
-
-        new_tokens = next_tokens[:i]
-        start_offset = insts[i].offset
-        new_tokens.append(
-            Token(
-                opname="COLLECTION_START",
-                attr=collection_enum,
-                pattr="CONST_MAP",
-                offset="%s_0" % start_offset,
-                linestart=insts[i].starts_line,
-                has_arg=True,
-                has_extended_arg=False,
-                opc=self.opc,
-                optype="pseudo",
-            )
-        )
-        for j in range(i + 1, collection_end, 3):
-            new_tokens.append(
-                Token(
-                    opname="ADD_KEY",
-                    attr=insts[j + 1].argval,
-                    pattr=insts[j + 1].argrepr,
-                    offset=insts[j + 1].offset,
-                    linestart=insts[j + 1].starts_line,
-                    has_arg=True,
-                    has_extended_arg=False,
-                    opc=self.opc,
-                    optype="pseudo",
-                )
-            )
-            new_tokens.append(
-                Token(
-                    opname="ADD_VALUE",
-                    attr=insts[j].argval,
-                    pattr=insts[j].argrepr,
-                    offset=insts[j].offset,
-                    linestart=insts[j].starts_line,
-                    has_arg=True,
-                    has_extended_arg=False,
-                    opc=self.opc,
-                    optype="pseudo",
-                )
-            )
-        new_tokens.append(
-            Token(
-                opname="BUILD_DICT_OLDER",
-                attr=t.attr,
-                pattr=t.pattr,
-                offset=t.offset,
-                linestart=t.linestart,
-                has_arg=t.has_arg,
-                has_extended_arg=False,
-                opc=t.opc,
-                optype="pseudo",
-            )
-        )
-        return new_tokens
-
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
         """
         Create "tokens" the bytecode of an Python code object. Largely these
@@ -575,6 +490,7 @@ class Scanner3(Scanner):
 
         last_op_was_break = False
         new_tokens = []
+
         skip_end_offset = None
 
         for i, inst in enumerate(self.insts):
@@ -621,36 +537,19 @@ class Scanner3(Scanner):
                     continue
 
             elif opname in ("BUILD_MAP",):
-                bound_map_from_insts_fn = (
+                if self.version >= (3, 5):
-                    self.bound_map_from_inst_35
+                    try_tokens = self.bound_map_from_inst_35(
-                    if self.version >= (3, 5)
-                    else self.bound_map_from_inst_pre35
-                )
-                try_tokens = bound_map_from_insts_fn(
                         self.insts,
                         new_tokens,
                         t,
                         i,
                     )
                     if try_tokens is not None:
-                    if self.version < (3, 5):
-                        assert try_tokens[-1] == "BUILD_DICT_OLDER"
-                        prev_offset = inst.offset
-                        for j in range(i, len(self.insts)):
-                            if self.insts[j].opname == "STORE_NAME":
-                                new_tokens = try_tokens
-                                skip_end_offset = prev_offset
-                                # Set a hacky sentinal to indicate skipping to the
-                                # next instruction
-                                opname = "EXTENDED_ARG"
-                                break
-                            prev_offset = self.insts[j].offset
-                            pass
-                        pass
-                    else:
                         new_tokens = try_tokens
                         continue
                         pass
+                    pass
+                pass
 
             argval = inst.argval
             op = inst.opcode

uncompyle6/semantics/fragments.py

@@ -1519,9 +1519,9 @@ class FragmentsWalker(pysource.SourceWalker, object):
             self.write("(")
             if kwargs:
                 # Last arg is tuple of keyword values: omit
-                l = n - 1
+                m = n - 1
             else:
-                l = n
+                m = n
 
             if kwargs:
                 # 3.6+ does this
@@ -1533,7 +1533,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                     j += 1
 
                 j = 0
-                while i < l:
+                while i < m:
                     self.write(sep)
                     value = self.traverse(node[i])
                     self.write("%s=%s" % (kwargs[j], value))
@@ -1541,7 +1541,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                     j += 1
                     i += 1
             else:
-                while i < l:
+                while i < m:
                     value = self.traverse(node[i])
                     i += 1
                     self.write(sep, value)
@@ -1793,12 +1793,12 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
     def template_engine(self, entry, startnode):
         """The format template interpretation engine.  See the comment at the
-        beginning of this module for the how we interpret format
+        beginning of this module for how we interpret format
         specifications such as %c, %C, and so on.
         """
 
         # print("-----")
-        # print(startnode)
+        # print(startnode.kind)
         # print(entry[0])
         # print('======')
 
@@ -1853,14 +1853,27 @@ class FragmentsWalker(pysource.SourceWalker, object):
 
                 index = entry[arg]
                 if isinstance(index, tuple):
+                    if isinstance(index[1], str):
+                        # if node[index[0]] != index[1]:
+                        #     from trepan.api import debug; debug()
                         assert (
                             node[index[0]] == index[1]
-                    ), "at %s[%d], expected %s node; got %s" % (
+                        ), "at %s[%d], expected '%s' node; got '%s'" % (
                             node.kind,
                             arg,
-                        node[index[0]].kind,
                             index[1],
+                            node[index[0]].kind,
                         )
+                    else:
+                        assert (
+                            node[index[0]] in index[1]
+                        ), "at %s[%d], expected to be in '%s' node; got '%s'" % (
+                            node.kind,
+                            arg,
+                            index[1],
+                            node[index[0]].kind,
+                        )
+
                     index = index[0]
                 assert isinstance(
                     index, int
@@ -1880,6 +1893,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 assert isinstance(tup, tuple)
                 if len(tup) == 3:
                     (index, nonterm_name, self.prec) = tup
+                    if isinstance(tup[1], str):
                         assert (
                             node[index] == nonterm_name
                         ), "at %s[%d], expected '%s' node; got '%s'" % (
@@ -1888,6 +1902,12 @@ class FragmentsWalker(pysource.SourceWalker, object):
                             nonterm_name,
                             node[index].kind,
                         )
+                    else:
+                        assert node[tup[0]] in tup[1], (
+                            f"at {node.kind}[{tup[0]}], expected to be in '{tup[1]}' "
+                            f"node; got '{node[tup[0]].kind}'"
+                        )
+
                 else:
                     assert len(tup) == 2
                     (index, self.prec) = entry[arg]
@@ -2098,6 +2118,7 @@ def code_deparse(
     # Build Syntax Tree from tokenized and massaged disassembly.
     # deparsed = pysource.FragmentsWalker(out, scanner, showast=showast)
     show_tree = debug_opts.get("tree", False)
+    linestarts = dict(scanner.opc.findlinestarts(co))
     deparsed = walker(
         version,
         scanner,