Merge pull request #222 from rocky/python-3.8

Python 3.8

Makefile

@@ -40,6 +40,9 @@ check-3.0 check-3.1 check-3.2 check-3.6:
 check-3.7: pytest
 	$(MAKE) -C test check
 
+check-3.8:
+	$(MAKE) -C test check
+
 #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0
 # Skip for now
 2.6 5.0 5.3 5.6 5.8:

__pkginfo__.py

@@ -23,7 +23,7 @@
 
 # Things that change more often go here.
 copyright   = """
-Copyright (C) 2015-2018 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2019 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
 classifiers =  ['Development Status :: 5 - Production/Stable',
@@ -57,7 +57,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
-                      'xdis >= 3.9.0, < 3.10.0']
+                      'xdis >= 4.0.0, < 4.1.0']
 
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'

setup.py

@@ -4,8 +4,8 @@ import sys
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 6) <= SYS_VERSION  <= (3, 7)):
+if not ((2, 6) <= SYS_VERSION  <= (3, 8)):
-    mess = "Python Release 2.6 .. 3.7 are supported in this code branch."
+    mess = "Python Release 2.6 .. 3.8 are supported in this code branch."
     if ((2, 4) <= SYS_VERSION <= (2, 7)):
         mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." %
                  sys.version[0:3])

test/Makefile

@@ -33,43 +33,48 @@ check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-na
 
 #: Run working tests from Python 3.0
 check-3.0: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.1
 check-3.1: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.2
 check-3.2: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.3
 check-3.3: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.6
 check-3.6: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.7
 check-3.7: check-bytecode
-	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
 	$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
+
+#: Run working tests from Python 3.8
+check-3.8: check-bytecode
+	$(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify $(COMPILE)
 
 # FIXME
 #: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
@@ -92,7 +97,8 @@ check-bytecode-2:
 check-bytecode-3:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 \
 	--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
-	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-3.7 \
+	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+        --bytecode-3.7 --bytecode-3.8 \
 	--bytecode-pypy3.2
 
 #: Check deparsing on selected bytecode 3.x
@@ -217,43 +223,43 @@ grammar-coverage-3.6:
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify
 
 #: Check deparsing Python 3.0
 check-bytecode-3.0:
-	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
 
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
-	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
 
 #: Check deparsing Python 3.2
 check-bytecode-3.2:
-	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify
 
 #: Check deparsing Python 3.3
 check-bytecode-3.3:
-	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify
 
 #: Check deparsing Python 3.4
 check-bytecode-3.4:
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify
 
 #: Check deparsing Python 3.5
 check-bytecode-3.5:
-	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify
 	$(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run
+	$(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify
 
 #: Check deparsing Python 3.6
 check-bytecode-3.6:
@@ -264,6 +270,10 @@ check-bytecode-3.6:
 check-bytecode-3.7:
 	$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify
 
+#: Check deparsing Python 3.8
+check-bytecode-3.8:
+	$(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify
+
 #: short tests for bytecodes only for this version of Python
 check-native-short:
 	$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE)

test/add-test.py

@@ -4,12 +4,19 @@
 import os, sys, py_compile
 assert len(sys.argv) >= 2
 version = sys.version[0:3]
-for path in sys.argv[1:]:
+if sys.argv[1] == '--run':
+    suffix = '_run'
+    py_source = sys.argv[2:]
+else:
+    suffix = ''
+    py_source = sys.argv[1:]
+
+for path in py_source:
     short = os.path.basename(path)
     if hasattr(sys, 'pypy_version_info'):
-        cfile =  "bytecode_pypy%s/%s" % (version, short) + 'c'
+        cfile =  "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c'
     else:
-        cfile =  "bytecode_%s/%s" % (version, short) + 'c'
+        cfile =  "bytecode_%s%s/%s" % (version, suffix, short) + 'c'
     print("byte-compiling %s to %s" % (path, cfile))
     py_compile.compile(path, cfile)
     if isinstance(version, str) or version >= (2, 6, 0):

test/simple_source/bug36/00_if_elif.py

@@ -1,3 +1,4 @@
+# Self-checking test.
 # Python 3 bug in not detecting the end bounds of if elif.
 def testit(b):
     if b == 1:

test/simple_source/bug36/01_fstring.py

@@ -1,4 +1,5 @@
-# Self-checking 3.6+ string interpolation tests
+# Self-checking test.
+# String interpolation tests
 
 var1 = 'x'
 var2 = 'y'

test/simple_source/looping/10_for.py

@@ -1,8 +1,17 @@
+# Self-checking test.
 # Tests:
-#  forstmt ::= SETUP_LOOP expr _for store
+#  for ::= SETUP_LOOP expr for_iter store
-#              for_block POP_BLOCK COME_FROM
+#          for_block POP_BLOCK COME_FROM
-for a in [1]:
+# In 3.8+
-    c = 2
+#  for ::= expr for_iter store
+#          for_block POP_BLOCK COME_FROM
 
-for a in range(2):
+c = 0
-    c = 2
+for a in [1]:
+    c += a
+assert c == 1, c
+
+for a in range(3):
+    c += a
+
+assert c == 4, c

test/simple_source/operation_logic/10_mixed_boolean.py

@@ -1,12 +1,26 @@
+# Self-checking test.
+# Mixed boolean expresions
+
 b = True
+assert b, 'b = True'
 c = False
+assert not c, 'c = False'
 d = True
 a = b and c or d
+assert a, 'b and c or d'
 a = (b or c) and d
+assert a, '(b or c) and d'
 a = b or c or d
+assert  a, 'b or c or d'
 a = b and c and d
+assert not a, 'b and c and d'
 a = b or c and d
+assert a
 a = b and (c or d)
+assert a
 a = b and c or d
+assert a
 a = (b or c and d) and b
+assert a
 a = (b or c and d or a) and b
+assert a

test/simple_source/slice/02_slice.py

@@ -1,8 +1,15 @@
+# Self-checking test.
 # Tests of Python slice operators
 
 ary = [1,2,3,4,5]
+
 # Forces BUILD_SLICE 2 on 3.x
-ary[:2]
+a = ary[:2]
-ary[2:]
+assert a == [1, 2]
+
+a = ary[2:]
+assert a == [3, 4, 5]
+
 # Forces BUILD_SLICE 3
-ary[1:4:2]
+a = ary[1:4:2]
+assert a == [2, 4]

test/test_pythonlib.py

@@ -81,7 +81,7 @@ for  vers in (2.7, 3.4, 3.5, 3.6):
 for  vers in (1.3, 1.4, 1.5,
               2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
               3.0, 3.1, 3.2, 3.3,
-              3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'):
+              3.4, 3.5, 3.6, 3.7, 3.8, 'pypy3.2', 'pypy2.7'):
     bytecode = "bytecode_%s" % vers
     key = "bytecode-%s" % vers
     test_options[key] =  (bytecode, PYC, bytecode, vers)

uncompyle6/bin/uncompile.py

@@ -72,9 +72,9 @@ def main_bin():
     if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0),
                                       (3, 1), (3, 2), (3, 3),
                                       (3, 4), (3, 5), (3, 6),
-                                      (3, 7)
+                                      (3, 7), (3, 8)
         )):
-        print('Error: %s requires Python 2.6-3.7' % program,
+        print('Error: %s requires Python 2.6-3.8' % program,
               file=sys.stderr)
         sys.exit(-1)
 

uncompyle6/parser.py

@@ -747,6 +747,12 @@ def get_python_parser(
                 p = parse37.Python37Parser(debug_parser)
             else:
                 p = parse37.Python37ParserSingle(debug_parser)
+        elif version == 3.8:
+            import uncompyle6.parsers.parse38 as parse38
+            if compile_mode == 'exec':
+                p = parse38.Python38Parser(debug_parser)
+            else:
+                p = parse38.Python38ParserSingle(debug_parser)
         else:
             if compile_mode == 'exec':
                 p = parse3.Python3Parser(debug_parser)

uncompyle6/parsers/parse3.py

@@ -1190,7 +1190,9 @@ class Python3Parser(PythonParser):
                 last += 1
             if last == n:
                 return False
-            return tokens[first].attr > tokens[last].offset
+            # 3.8+ Doesn't have SETUP_LOOP
+            return self.version < 3.8 and tokens[first].attr > tokens[last].offset
+
         elif rule == ('try_except',
                       ('SETUP_EXCEPT', 'suite_stmts_opt', 'POP_BLOCK',
                        'except_handler', 'opt_come_from_except')):

uncompyle6/parsers/parse36.py

@@ -231,8 +231,6 @@ class Python36Parser(Python35Parser):
                 self.addRule(rule, nop_func)
             elif opname == 'SETUP_WITH':
                 rules_str = """
-                withstmt   ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
-                               WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
                 withstmt   ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
                                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
@@ -240,6 +238,19 @@ class Python36Parser(Python35Parser):
                 withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
                                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
                 """
+                if self.version < 3.8:
+                    rules_str += """
+                    withstmt   ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   LOAD_CONST
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                    """
+                else:
+                    rules_str += """
+                    withstmt   ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
+                                   BEGIN_FINALLY COME_FROM_WITH
+                                   WITH_CLEANUP_START WITH_CLEANUP_FINISH
+                                   END_FINALLY
+                    """
                 self.addRule(rules_str, nop_func)
                 pass
             pass

uncompyle6/parsers/parse38.py

@@ -0,0 +1,95 @@
+#  Copyright (c) 2017-2019 Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+spark grammar differences over Python 3.7 for Python 3.8
+"""
+from __future__ import print_function
+
+from uncompyle6.parser import PythonParserSingle
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parsers.parse37 import Python37Parser
+
+class Python38Parser(Python37Parser):
+
+    def p_38misc(self, args):
+        """
+        stmt               ::= for38
+        stmt               ::= forelsestmt38
+        stmt               ::= forelselaststmt38
+        stmt               ::= forelselaststmtl38
+
+        for38              ::= expr get_iter store for_block JUMP_BACK
+        for38              ::= expr for_iter store for_block JUMP_BACK
+        for38              ::= expr for_iter store for_block JUMP_BACK POP_BLOCK
+
+        forelsestmt38      ::= expr for_iter store for_block POP_BLOCK else_suite
+        forelselaststmt38  ::= expr for_iter store for_block POP_BLOCK else_suitec
+        forelselaststmtl38 ::= expr for_iter store for_block POP_BLOCK else_suitel
+        whilestmt          ::= testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
+        whilestmt          ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+        whilestmt          ::= testexpr returns          POP_BLOCK
+        while1elsestmt     ::=          l_stmts     JUMP_BACK
+        whileelsestmt      ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+        whileTruestmt      ::= l_stmts_opt          JUMP_BACK POP_BLOCK
+        while1stmt         ::= l_stmts COME_FROM_LOOP
+        while1stmt         ::= l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+        while1elsestmt     ::= l_stmts JUMP_BACK
+        whileTruestmt      ::= l_stmts_opt JUMP_BACK NOP
+        whileTruestmt      ::= l_stmts_opt JUMP_BACK POP_BLOCK NOP
+        for_block          ::= l_stmts_opt _come_from_loops JUMP_BACK
+        """
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python38Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+    def customize_grammar_rules(self, tokens, customize):
+        self.remove_rules("""
+           stmt               ::= for
+           stmt               ::= forelsestmt
+           for                ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
+           for                ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP
+           for_block          ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK
+           forelsestmt38      ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite
+           forelselaststmt38  ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec
+           forelselaststmtl38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel
+
+        """)
+        super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+
+class Python38ParserSingle(Python38Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python38Parser()
+    p.check_grammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.8:
+        lhs, rhs, tokens, right_recursive = p.check_sets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        # print(sorted(p.rule2name.items()))

uncompyle6/scanner.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016, 2018 by Rocky Bernstein
+#  Copyright (c) 2016, 2018-2019 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -39,7 +39,7 @@ from xdis.util import code2num
 # Note: these all have to be floats
 PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5,
                              2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
-                             3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7))
+                             3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8))
 
 CANONIC2VERSION = dict((canonic_python_version[str(v)], v) for v in PYTHON_VERSIONS)
 

uncompyle6/scanners/scanner13.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2018 by Rocky Bernstein
+#  Copyright (c) 2018-2019 by Rocky Bernstein
 """
 Python 1.3 bytecode decompiler massaging.
 
@@ -25,7 +25,7 @@ class Scanner13(scan.Scanner14):
         self.version = 1.3
         return
 
-    # def ingest22(self, co, classname=None, code_objects={}, show_asm=None):
+    # def ingest(self, co, classname=None, code_objects={}, show_asm=None):
     #     tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm)
     #     tokens = [t for t in tokens if t.kind != 'SET_LINENO']
 

uncompyle6/scanners/scanner14.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2018 by Rocky Bernstein
+#  Copyright (c) 2018-2019 by Rocky Bernstein
 """
 Python 1.4 bytecode decompiler massaging.
 
@@ -26,7 +26,7 @@ class Scanner14(scan.Scanner15):
         self.genexpr_name = '<generator expression>'
         return
 
-    # def ingest22(self, co, classname=None, code_objects={}, show_asm=None):
+    # def ingest(self, co, classname=None, code_objects={}, show_asm=None):
     #     tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm)
     #     tokens = [t for t in tokens if t.kind != 'SET_LINENO']
 

uncompyle6/scanners/scanner3.py

@@ -64,8 +64,13 @@ class Scanner3(Scanner):
         # Ops that start SETUP_ ... We will COME_FROM with these names
         # Some blocks and END_ statements. And they can start
         # a new statement
-        setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT,
+        if self.version < 3.8:
-                      self.opc.SETUP_FINALLY]
+            setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT,
+                         self.opc.SETUP_FINALLY]
+            self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP])
+        else:
+            setup_ops = [self.opc.SETUP_FINALLY]
+            self.setup_ops_no_loop = frozenset(setup_ops)
 
         if self.version >= 3.2:
             setup_ops.append(self.opc.SETUP_WITH)
@@ -78,11 +83,9 @@ class Scanner3(Scanner):
             self.pop_jump_tf = frozenset([self.opc.PJIF, self.opc.PJIT])
             self.not_continue_follow = ('END_FINALLY', 'POP_BLOCK')
 
-        self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP])
 
         # Opcodes that can start a statement.
         statement_opcodes = [
-            self.opc.BREAK_LOOP,    self.opc.CONTINUE_LOOP,
             self.opc.POP_BLOCK,     self.opc.STORE_FAST,
             self.opc.DELETE_FAST,   self.opc.STORE_DEREF,
 
@@ -97,6 +100,9 @@ class Scanner3(Scanner):
             self.opc.PRINT_EXPR,   self.opc.JUMP_ABSOLUTE
         ]
 
+        if self.version < 3.8:
+            statement_opcodes += [self.opc.BREAK_LOOP, self.opc.CONTINUE_LOOP]
+
         self.statement_opcodes = frozenset(statement_opcodes) | self.setup_ops_no_loop
 
         # Opcodes that can start a "store" non-terminal.
@@ -628,7 +634,7 @@ class Scanner3(Scanner):
                 end    = current_end
                 parent = struct
 
-        if op == self.opc.SETUP_LOOP:
+        if self.version < 3.8 and op == self.opc.SETUP_LOOP:
             # We categorize loop types: 'for', 'while', 'while 1' with
             # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
@@ -857,6 +863,11 @@ class Scanner3(Scanner):
             # if the condition jump is to a forward location.
             # Also the existence of a jump to the instruction after "END_FINALLY"
             # will distinguish "try/else" from "try".
+            if self.version < 3.8:
+                rtarget_break = (self.opc.RETURN_VALUE, self.opc.BREAK_LOOP)
+            else:
+                rtarget_break = (self.opc.RETURN_VALUE,)
+
             if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5):
                 if_end = self.get_target(pre_rtarget)
 
@@ -891,8 +902,7 @@ class Scanner3(Scanner):
                                      'start': start,
                                      'end': pre_rtarget})
                 self.not_continue.add(pre_rtarget)
-            elif code[pre_rtarget] in (self.opc.RETURN_VALUE,
+            elif code[pre_rtarget] in rtarget_break:
-                                       self.opc.BREAK_LOOP):
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -957,7 +967,7 @@ class Scanner3(Scanner):
                     if rtarget > offset:
                         self.fixed_jumps[offset] = rtarget
 
-        elif op == self.opc.SETUP_EXCEPT:
+        elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT:
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
             self.fixed_jumps[offset] = end

uncompyle6/scanners/scanner38.py

@@ -0,0 +1,51 @@
+#  Copyright (c) 2019 by Rocky Bernstein
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Python 3.8 bytecode decompiler scanner
+
+Does some additional massaging of xdis-disassembled instructions to
+make things easier for decompilation.
+
+This sets up opcodes Python's 3.8 and calls a generalized
+scanner routine for Python 3.
+"""
+
+from __future__ import print_function
+
+from uncompyle6.scanners.scanner3 import Scanner3
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_38 as opc
+JUMP_OPs = opc.JUMP_OPS
+
+class Scanner38(Scanner3):
+
+    def __init__(self, show_asm=None):
+        Scanner3.__init__(self, 3.8, show_asm)
+        return
+    pass
+
+if __name__ == "__main__":
+    from uncompyle6 import PYTHON_VERSION
+    if PYTHON_VERSION == 3.8:
+        import inspect
+        co = inspect.currentframe().f_code
+        tokens, customize = Scanner38().ingest(co)
+        for t in tokens:
+            print(t.format())
+        pass
+    else:
+        print("Need to be Python 3.8 to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/semantics/consts.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017, 2018 by Rocky Bernstein
+#  Copyright (c) 2017-2019 by Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -276,13 +276,28 @@ TABLE_DIRECT = {
     'while1elsestmt':   ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ),
     'whileelsestmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ),
     'whileelselaststmt':	( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ),
-    'for':              ( '%|for %c in %c:\n%+%c%-\n\n', (3, 'store'), 1, 4 ),
+
+    # Note: Python 3.8+ changes this
+    'for':              ( '%|for %c in %c:\n%+%c%-\n\n',
+                          (3, 'store'),
+                          (1, 'expr'),
+                          (4, 'for_block') ),
     'forelsestmt':	    (
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', (3, 'store'), 1, 4, -2 ),
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+                          (3, 'store'),
+                          (1, 'expr'),
+                          (4, 'for_block'), -2 ),
     'forelselaststmt':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', (3, 'store'), 1, 4, -2 ),
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-',
+                          (3, 'store'),
+                          (1, 'expr'),
+                          (4, 'for_block'), -2 ),
     'forelselaststmtl':	(
-        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', (3, 'store'), 1, 4, -2 ),
+        '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+                          (3, 'store'),
+                          (1, 'expr'),
+                          (4, 'for_block'), -2 ),
+
     'try_except':       ( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
     'tryelsestmt':	    ( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ),
     'tryelsestmtc':	    ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ),

uncompyle6/semantics/customize3.py

@@ -908,6 +908,10 @@ def customize_for_version3(self, version):
             self.n_return_closure = return_closure
 
             if version >= 3.7:
+                ########################
+                # Python 3.7+ changes
+                #######################
+
                 PRECEDENCE['attribute37'] = 2
                 TABLE_DIRECT.update({
                     'attribute37':  ( '%c.%[1]{pattr}', 0 ),
@@ -928,7 +932,37 @@ def customize_for_version3(self, version):
                     'compare_chained2b_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)),
 
                     })
+                if version >= 3.8:
+                    ########################
+                    # Python 3.8+ changes
+                    #######################
+                    for lhs in 'for forelsestmt forelselaststmt forelselaststmtl'.split():
+                        del TABLE_DIRECT[lhs]
+                    TABLE_DIRECT.update({
+                        'for38':            (
+                            '%|for %c in %c:\n%+%c%-\n\n',
+                            (2, 'store'),
+                            (0, 'expr'),
+                            (3, 'for_block') ),
+                        'forelsestmt38':    (
+                            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+                            (2, 'store'),
+                            (0, 'expr'),
+                            (3, 'for_block'), -2 ),
+                        'forelselaststmt38': (
+                            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-',
+                            (2, 'store'),
+                            (0, 'expr'),
+                            (3, 'for_block'), -2 ),
+                        'forelselaststmtl38':	(
+                            '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
+                            (2, 'store'),
+                            (0, 'expr'),
+                            (3, 'for_block'), -2 ),
+                    })
+                    pass
                 pass
+
             pass # version >= 3.6
         pass # version >= 3.4
     return