[WIP] - move forward a tad on Python 3.8

uncompyle6/parsers/parse38.py

@@ -0,0 +1,54 @@
+#  Copyright (c) 2017-2019 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+spark grammar differences over Python 3.7 for Python 3.8
+"""
+from __future__ import print_function
+
+from uncompyle6.parser import PythonParserSingle
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parsers.parse37 import Python37Parser
+
+class Python38Parser(Python37Parser):
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python38Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+
+class Python38ParserSingle(Python38Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python38Parser()
+    p.check_grammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.8:
+        lhs, rhs, tokens, right_recursive = p.check_sets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        # print(sorted(p.rule2name.items()))

uncompyle6/scanner.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016, 2018 by Rocky Bernstein
+#  Copyright (c) 2016, 2018-2019 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -39,7 +39,7 @@ from xdis.util import code2num
 # Note: these all have to be floats
 PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5,
                              2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
-                             3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7))
+                             3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8))
 
 CANONIC2VERSION = dict((canonic_python_version[str(v)], v) for v in PYTHON_VERSIONS)
 

uncompyle6/scanners/scanner3.py

@@ -64,8 +64,13 @@ class Scanner3(Scanner):
         # Ops that start SETUP_ ... We will COME_FROM with these names
         # Some blocks and END_ statements. And they can start
         # a new statement
+        if self.version < 3.8:
             setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT,
                          self.opc.SETUP_FINALLY]
+            self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP])
+        else:
+            setup_ops = [self.opc.SETUP_FINALLY]
+            self.setup_ops_no_loop = frozenset(setup_ops)
 
         if self.version >= 3.2:
             setup_ops.append(self.opc.SETUP_WITH)
@@ -78,11 +83,9 @@ class Scanner3(Scanner):
             self.pop_jump_tf = frozenset([self.opc.PJIF, self.opc.PJIT])
             self.not_continue_follow = ('END_FINALLY', 'POP_BLOCK')
 
-        self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP])
 
         # Opcodes that can start a statement.
         statement_opcodes = [
-            self.opc.BREAK_LOOP,    self.opc.CONTINUE_LOOP,
             self.opc.POP_BLOCK,     self.opc.STORE_FAST,
             self.opc.DELETE_FAST,   self.opc.STORE_DEREF,
 
@@ -97,6 +100,9 @@ class Scanner3(Scanner):
             self.opc.PRINT_EXPR,   self.opc.JUMP_ABSOLUTE
         ]
 
+        if self.version < 3.8:
+            statement_opcodes += [self.opc.BREAK_LOOP, self.opc.CONTINUE_LOOP]
+
         self.statement_opcodes = frozenset(statement_opcodes) | self.setup_ops_no_loop
 
         # Opcodes that can start a "store" non-terminal.
@@ -628,7 +634,7 @@ class Scanner3(Scanner):
                 end    = current_end
                 parent = struct
 
-        if op == self.opc.SETUP_LOOP:
+        if self.version < 3.8 and op == self.opc.SETUP_LOOP:
             # We categorize loop types: 'for', 'while', 'while 1' with
             # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
@@ -857,6 +863,11 @@ class Scanner3(Scanner):
             # if the condition jump is to a forward location.
             # Also the existence of a jump to the instruction after "END_FINALLY"
             # will distinguish "try/else" from "try".
+            if self.version < 3.8:
+                rtarget_break = (self.opc.RETURN_VALUE, self.opc.BREAK_LOOP)
+            else:
+                rtarget_break = (self.opc.RETURN_VALUE,)
+
             if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5):
                 if_end = self.get_target(pre_rtarget)
 
@@ -891,8 +902,7 @@ class Scanner3(Scanner):
                                      'start': start,
                                      'end': pre_rtarget})
                 self.not_continue.add(pre_rtarget)
-            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+            elif code[pre_rtarget] in rtarget_break:
-                                       self.opc.BREAK_LOOP):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -957,7 +967,7 @@ class Scanner3(Scanner):
                     if rtarget > offset:
                         self.fixed_jumps[offset] = rtarget
 
-        elif op == self.opc.SETUP_EXCEPT:
+        elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT:
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
             self.fixed_jumps[offset] = end

uncompyle6/scanners/scanner38.py

@@ -0,0 +1,51 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Python 3.8 bytecode decompiler scanner
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+
+This sets up opcodes Python's 3.8 and calls a generalized
+scanner routine for Python 3.
+"""
+
+from __future__ import print_function
+
+from uncompyle6.scanners.scanner37 import Scanner3
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_37 as opc
+JUMP_OPs = opc.JUMP_OPS
+
+class Scanner38(Scanner3):
+
+    def __init__(self, show_asm=None):
+        Scanner3.__init__(self, 3.8, show_asm)
+        return
+    pass
+
+if __name__ == "__main__":
+    from uncompyle6 import PYTHON_VERSION
+    if PYTHON_VERSION == 3.8:
+        import inspect
+        co = inspect.currentframe().f_code
+        tokens, customize = Scanner38().ingest(co)
+        for t in tokens:
+            print(t.format())
+        pass
+    else:
+        print("Need to be Python 3.8 to demo; I am %s." %
+              PYTHON_VERSION)