diff --git a/Makefile b/Makefile index 5fe5d565..21d91675 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,9 @@ check-3.0 check-3.1 check-3.2 check-3.6: check-3.7: pytest $(MAKE) -C test check +check-3.8: + $(MAKE) -C test check + #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0 # Skip for now 2.6 5.0 5.3 5.6 5.8: diff --git a/__pkginfo__.py b/__pkginfo__.py index 135da258..39fccce6 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -23,7 +23,7 @@ # Things that change more often go here. copyright = """ -Copyright (C) 2015-2018 Rocky Bernstein . +Copyright (C) 2015-2019 Rocky Bernstein . """ classifiers = ['Development Status :: 5 - Production/Stable', @@ -57,7 +57,7 @@ entry_points = { ]} ftp_url = None install_requires = ['spark-parser >= 1.8.7, < 1.9.0', - 'xdis >= 3.9.0, < 3.10.0'] + 'xdis >= 4.0.0, < 4.1.0'] license = 'GPL3' mailing_list = 'python-debugger@googlegroups.com' diff --git a/setup.py b/setup.py index 59953230..dc120134 100755 --- a/setup.py +++ b/setup.py @@ -4,8 +4,8 @@ import sys """Setup script for the 'uncompyle6' distribution.""" SYS_VERSION = sys.version_info[0:2] -if not ((2, 6) <= SYS_VERSION <= (3, 7)): - mess = "Python Release 2.6 .. 3.7 are supported in this code branch." +if not ((2, 6) <= SYS_VERSION <= (3, 8)): + mess = "Python Release 2.6 .. 3.8 are supported in this code branch." if ((2, 4) <= SYS_VERSION <= (2, 7)): mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." % sys.version[0:3]) diff --git a/test/Makefile b/test/Makefile index 6d12cf0a..91c0a7e4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -33,43 +33,48 @@ check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-na #: Run working tests from Python 3.0 check-3.0: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE) #: Run working tests from Python 3.1 check-3.1: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE) #: Run working tests from Python 3.2 check-3.2: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE) #: Run working tests from Python 3.3 check-3.3: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE) #: Run working tests from Python 3.4 check-3.4: check-bytecode check-3.4-ok check-2.7-ok - $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE) #: Run working tests from Python 3.5 check-3.5: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE) #: Run working tests from Python 3.6 check-3.6: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE) #: Run working tests from Python 3.7 check-3.7: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE) + +#: Run working tests from Python 3.8 +check-3.8: check-bytecode + $(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify $(COMPILE) # FIXME #: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0 @@ -92,7 +97,8 @@ check-bytecode-2: check-bytecode-3: $(PYTHON) test_pythonlib.py --bytecode-3.0 \ --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \ - --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-3.7 \ + --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \ + --bytecode-3.7 --bytecode-3.8 \ --bytecode-pypy3.2 #: Check deparsing on selected bytecode 3.x @@ -217,43 +223,43 @@ grammar-coverage-3.6: #: Check deparsing Python 2.6 check-bytecode-2.6: - $(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify #: Check deparsing Python 2.7 check-bytecode-2.7: - $(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify #: Check deparsing Python 3.0 check-bytecode-3.0: - $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify #: Check deparsing Python 3.1 check-bytecode-3.1: - $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify #: Check deparsing Python 3.2 check-bytecode-3.2: - $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify #: Check deparsing Python 3.3 check-bytecode-3.3: - $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify #: Check deparsing Python 3.4 check-bytecode-3.4: - $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify #: Check deparsing Python 3.5 check-bytecode-3.5: - $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify #: Check deparsing Python 3.6 check-bytecode-3.6: @@ -264,6 +270,10 @@ check-bytecode-3.6: check-bytecode-3.7: $(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify +#: Check deparsing Python 3.8 +check-bytecode-3.8: + $(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify + #: short tests for bytecodes only for this version of Python check-native-short: $(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE) diff --git a/test/add-test.py b/test/add-test.py index 905323d5..a5ea9ac6 100755 --- a/test/add-test.py +++ b/test/add-test.py @@ -4,12 +4,19 @@ import os, sys, py_compile assert len(sys.argv) >= 2 version = sys.version[0:3] -for path in sys.argv[1:]: +if sys.argv[1] == '--run': + suffix = '_run' + py_source = sys.argv[2:] +else: + suffix = '' + py_source = sys.argv[1:] + +for path in py_source: short = os.path.basename(path) if hasattr(sys, 'pypy_version_info'): - cfile = "bytecode_pypy%s/%s" % (version, short) + 'c' + cfile = "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c' else: - cfile = "bytecode_%s/%s" % (version, short) + 'c' + cfile = "bytecode_%s%s/%s" % (version, suffix, short) + 'c' print("byte-compiling %s to %s" % (path, cfile)) py_compile.compile(path, cfile) if isinstance(version, str) or version >= (2, 6, 0): diff --git a/test/bytecode_2.7_run/02_slice.pyc b/test/bytecode_2.7_run/02_slice.pyc new file mode 100644 index 00000000..f3fd0939 Binary files /dev/null and b/test/bytecode_2.7_run/02_slice.pyc differ diff --git a/test/bytecode_3.2/02_slice.pyc b/test/bytecode_3.2/02_slice.pyc deleted file mode 100644 index da9ccee1..00000000 Binary files a/test/bytecode_3.2/02_slice.pyc and /dev/null differ diff --git a/test/bytecode_3.2_run/02_slice.pyc b/test/bytecode_3.2_run/02_slice.pyc new file mode 100644 index 00000000..b7c23102 Binary files /dev/null and b/test/bytecode_3.2_run/02_slice.pyc differ diff --git a/test/bytecode_3.3/02_slice.pyc b/test/bytecode_3.3/02_slice.pyc deleted file mode 100644 index 0d1a3d43..00000000 Binary files a/test/bytecode_3.3/02_slice.pyc and /dev/null differ diff --git a/test/bytecode_3.3/10_for.pyc b/test/bytecode_3.3/10_for.pyc deleted file mode 100644 index 3379248f..00000000 Binary files a/test/bytecode_3.3/10_for.pyc and /dev/null differ diff --git a/test/bytecode_3.3/10_mixed_boolean.pyc b/test/bytecode_3.3/10_mixed_boolean.pyc deleted file mode 100644 index ecd44155..00000000 Binary files a/test/bytecode_3.3/10_mixed_boolean.pyc and /dev/null differ diff --git a/test/bytecode_3.3_run/02_slice.pyc b/test/bytecode_3.3_run/02_slice.pyc new file mode 100644 index 00000000..94d9b3e3 Binary files /dev/null and b/test/bytecode_3.3_run/02_slice.pyc differ diff --git a/test/bytecode_3.3_run/10_for.pyc b/test/bytecode_3.3_run/10_for.pyc new file mode 100644 index 00000000..f528dfb8 Binary files /dev/null and b/test/bytecode_3.3_run/10_for.pyc differ diff --git a/test/bytecode_3.3_run/10_mixed_boolean.pyc b/test/bytecode_3.3_run/10_mixed_boolean.pyc new file mode 100644 index 00000000..ba4fdeea Binary files /dev/null and b/test/bytecode_3.3_run/10_mixed_boolean.pyc differ diff --git a/test/bytecode_3.4/02_slice.pyc b/test/bytecode_3.4/02_slice.pyc deleted file mode 100644 index fbf24a37..00000000 Binary files a/test/bytecode_3.4/02_slice.pyc and /dev/null differ diff --git a/test/bytecode_3.4/10_for.pyc b/test/bytecode_3.4/10_for.pyc deleted file mode 100644 index 26c63996..00000000 Binary files a/test/bytecode_3.4/10_for.pyc and /dev/null differ diff --git a/test/bytecode_3.4/10_mixed_boolean.pyc b/test/bytecode_3.4/10_mixed_boolean.pyc deleted file mode 100644 index ca6b6d6b..00000000 Binary files a/test/bytecode_3.4/10_mixed_boolean.pyc and /dev/null differ diff --git a/test/bytecode_3.4_run/02_slice.pyc b/test/bytecode_3.4_run/02_slice.pyc new file mode 100644 index 00000000..f06f157a Binary files /dev/null and b/test/bytecode_3.4_run/02_slice.pyc differ diff --git a/test/bytecode_3.4_run/10_for.pyc b/test/bytecode_3.4_run/10_for.pyc new file mode 100644 index 00000000..ee588b94 Binary files /dev/null and b/test/bytecode_3.4_run/10_for.pyc differ diff --git a/test/bytecode_3.4_run/10_mixed_boolean.pyc b/test/bytecode_3.4_run/10_mixed_boolean.pyc new file mode 100644 index 00000000..0bf0fbfd Binary files /dev/null and b/test/bytecode_3.4_run/10_mixed_boolean.pyc differ diff --git a/test/bytecode_3.5/02_slice.pyc b/test/bytecode_3.5/02_slice.pyc deleted file mode 100644 index f8849007..00000000 Binary files a/test/bytecode_3.5/02_slice.pyc and /dev/null differ diff --git a/test/bytecode_3.5/10_for.pyc b/test/bytecode_3.5/10_for.pyc deleted file mode 100644 index 82d64337..00000000 Binary files a/test/bytecode_3.5/10_for.pyc and /dev/null differ diff --git a/test/bytecode_3.5/10_mixed_boolean.pyc b/test/bytecode_3.5/10_mixed_boolean.pyc deleted file mode 100644 index d8be2198..00000000 Binary files a/test/bytecode_3.5/10_mixed_boolean.pyc and /dev/null differ diff --git a/test/bytecode_3.5_run/02_slice.pyc b/test/bytecode_3.5_run/02_slice.pyc new file mode 100644 index 00000000..6e10051d Binary files /dev/null and b/test/bytecode_3.5_run/02_slice.pyc differ diff --git a/test/bytecode_3.5_run/10_for.pyc b/test/bytecode_3.5_run/10_for.pyc new file mode 100644 index 00000000..e669db77 Binary files /dev/null and b/test/bytecode_3.5_run/10_for.pyc differ diff --git a/test/bytecode_3.5_run/10_mixed_boolean.pyc b/test/bytecode_3.5_run/10_mixed_boolean.pyc new file mode 100644 index 00000000..76f64298 Binary files /dev/null and b/test/bytecode_3.5_run/10_mixed_boolean.pyc differ diff --git a/test/bytecode_3.6_run/02_slice.pyc b/test/bytecode_3.6_run/02_slice.pyc new file mode 100644 index 00000000..1fb04219 Binary files /dev/null and b/test/bytecode_3.6_run/02_slice.pyc differ diff --git a/test/bytecode_3.8/09_yield_from.pyc b/test/bytecode_3.8/09_yield_from.pyc new file mode 100644 index 00000000..8da7c6ad Binary files /dev/null and b/test/bytecode_3.8/09_yield_from.pyc differ diff --git a/test/bytecode_3.8_run/10_for.pyc b/test/bytecode_3.8_run/10_for.pyc new file mode 100644 index 00000000..38a345ee Binary files /dev/null and b/test/bytecode_3.8_run/10_for.pyc differ diff --git a/test/simple_source/bug36/00_if_elif.py b/test/simple_source/bug36/00_if_elif.py index 0867a01f..747ec4bc 100644 --- a/test/simple_source/bug36/00_if_elif.py +++ b/test/simple_source/bug36/00_if_elif.py @@ -1,3 +1,4 @@ +# Self-checking test. # Python 3 bug in not detecting the end bounds of if elif. def testit(b): if b == 1: diff --git a/test/simple_source/bug36/01_fstring.py b/test/simple_source/bug36/01_fstring.py index 26bc3904..c03aaa79 100644 --- a/test/simple_source/bug36/01_fstring.py +++ b/test/simple_source/bug36/01_fstring.py @@ -1,4 +1,5 @@ -# Self-checking 3.6+ string interpolation tests +# Self-checking test. +# String interpolation tests var1 = 'x' var2 = 'y' diff --git a/test/simple_source/looping/10_for.py b/test/simple_source/looping/10_for.py index eb85f549..5de22cab 100644 --- a/test/simple_source/looping/10_for.py +++ b/test/simple_source/looping/10_for.py @@ -1,8 +1,17 @@ +# Self-checking test. # Tests: -# forstmt ::= SETUP_LOOP expr _for store -# for_block POP_BLOCK COME_FROM -for a in [1]: - c = 2 +# for ::= SETUP_LOOP expr for_iter store +# for_block POP_BLOCK COME_FROM +# In 3.8+ +# for ::= expr for_iter store +# for_block POP_BLOCK COME_FROM -for a in range(2): - c = 2 +c = 0 +for a in [1]: + c += a +assert c == 1, c + +for a in range(3): + c += a + +assert c == 4, c diff --git a/test/simple_source/operation_logic/10_mixed_boolean.py b/test/simple_source/operation_logic/10_mixed_boolean.py index d0851a63..d1849cda 100644 --- a/test/simple_source/operation_logic/10_mixed_boolean.py +++ b/test/simple_source/operation_logic/10_mixed_boolean.py @@ -1,12 +1,26 @@ +# Self-checking test. +# Mixed boolean expresions + b = True +assert b, 'b = True' c = False +assert not c, 'c = False' d = True a = b and c or d +assert a, 'b and c or d' a = (b or c) and d +assert a, '(b or c) and d' a = b or c or d +assert a, 'b or c or d' a = b and c and d +assert not a, 'b and c and d' a = b or c and d +assert a a = b and (c or d) +assert a a = b and c or d +assert a a = (b or c and d) and b +assert a a = (b or c and d or a) and b +assert a diff --git a/test/simple_source/slice/02_slice.py b/test/simple_source/slice/02_slice.py index 569626b6..917b3837 100644 --- a/test/simple_source/slice/02_slice.py +++ b/test/simple_source/slice/02_slice.py @@ -1,8 +1,15 @@ +# Self-checking test. # Tests of Python slice operators ary = [1,2,3,4,5] + # Forces BUILD_SLICE 2 on 3.x -ary[:2] -ary[2:] +a = ary[:2] +assert a == [1, 2] + +a = ary[2:] +assert a == [3, 4, 5] + # Forces BUILD_SLICE 3 -ary[1:4:2] +a = ary[1:4:2] +assert a == [2, 4] diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index e74d9d46..df05fcf5 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -81,7 +81,7 @@ for vers in (2.7, 3.4, 3.5, 3.6): for vers in (1.3, 1.4, 1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, - 3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'): + 3.4, 3.5, 3.6, 3.7, 3.8, 'pypy3.2', 'pypy2.7'): bytecode = "bytecode_%s" % vers key = "bytecode-%s" % vers test_options[key] = (bytecode, PYC, bytecode, vers) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index 1151520a..c482bcbd 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -72,9 +72,9 @@ def main_bin(): if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6), - (3, 7) + (3, 7), (3, 8) )): - print('Error: %s requires Python 2.6-3.7' % program, + print('Error: %s requires Python 2.6-3.8' % program, file=sys.stderr) sys.exit(-1) diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 08d701c0..e7e5675c 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -747,6 +747,12 @@ def get_python_parser( p = parse37.Python37Parser(debug_parser) else: p = parse37.Python37ParserSingle(debug_parser) + elif version == 3.8: + import uncompyle6.parsers.parse38 as parse38 + if compile_mode == 'exec': + p = parse38.Python38Parser(debug_parser) + else: + p = parse38.Python38ParserSingle(debug_parser) else: if compile_mode == 'exec': p = parse3.Python3Parser(debug_parser) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index eacabdcd..98321047 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1190,7 +1190,9 @@ class Python3Parser(PythonParser): last += 1 if last == n: return False - return tokens[first].attr > tokens[last].offset + # 3.8+ Doesn't have SETUP_LOOP + return self.version < 3.8 and tokens[first].attr > tokens[last].offset + elif rule == ('try_except', ('SETUP_EXCEPT', 'suite_stmts_opt', 'POP_BLOCK', 'except_handler', 'opt_come_from_except')): diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 84a696b0..05ab5829 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -231,8 +231,6 @@ class Python36Parser(Python35Parser): self.addRule(rule, nop_func) elif opname == 'SETUP_WITH': rules_str = """ - withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST - WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY @@ -240,6 +238,19 @@ class Python36Parser(Python35Parser): withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY """ + if self.version < 3.8: + rules_str += """ + withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + LOAD_CONST + WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + """ + else: + rules_str += """ + withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_WITH + WITH_CLEANUP_START WITH_CLEANUP_FINISH + END_FINALLY + """ self.addRule(rules_str, nop_func) pass pass diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py new file mode 100644 index 00000000..5594f2c8 --- /dev/null +++ b/uncompyle6/parsers/parse38.py @@ -0,0 +1,95 @@ +# Copyright (c) 2017-2019 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +spark grammar differences over Python 3.7 for Python 3.8 +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parsers.parse37 import Python37Parser + +class Python38Parser(Python37Parser): + + def p_38misc(self, args): + """ + stmt ::= for38 + stmt ::= forelsestmt38 + stmt ::= forelselaststmt38 + stmt ::= forelselaststmtl38 + + for38 ::= expr get_iter store for_block JUMP_BACK + for38 ::= expr for_iter store for_block JUMP_BACK + for38 ::= expr for_iter store for_block JUMP_BACK POP_BLOCK + + forelsestmt38 ::= expr for_iter store for_block POP_BLOCK else_suite + forelselaststmt38 ::= expr for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl38 ::= expr for_iter store for_block POP_BLOCK else_suitel + whilestmt ::= testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK + whilestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whilestmt ::= testexpr returns POP_BLOCK + while1elsestmt ::= l_stmts JUMP_BACK + whileelsestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK + while1stmt ::= l_stmts COME_FROM_LOOP + while1stmt ::= l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + while1elsestmt ::= l_stmts JUMP_BACK + whileTruestmt ::= l_stmts_opt JUMP_BACK NOP + whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK NOP + for_block ::= l_stmts_opt _come_from_loops JUMP_BACK + """ + + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python38Parser, self).__init__(debug_parser) + self.customized = {} + + def customize_grammar_rules(self, tokens, customize): + self.remove_rules(""" + stmt ::= for + stmt ::= forelsestmt + for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK + for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP + for_block ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK + forelsestmt38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite + forelselaststmt38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel + + """) + super(Python37Parser, self).customize_grammar_rules(tokens, customize) + +class Python38ParserSingle(Python38Parser, PythonParserSingle): + pass + +if __name__ == '__main__': + # Check grammar + p = Python38Parser() + p.check_grammar() + from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.8: + lhs, rhs, tokens, right_recursive = p.check_sets() + from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) + opcode_set = set(s.opc.opname).union(set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split())) + remain_tokens = set(tokens) - opcode_set + import re + remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) + remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + remain_tokens = set(remain_tokens) - opcode_set + print(remain_tokens) + # print(sorted(p.rule2name.items())) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index c9318d8b..43bdf151 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, 2018 by Rocky Bernstein +# Copyright (c) 2016, 2018-2019 by Rocky Bernstein # Copyright (c) 2005 by Dan Pascu # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock @@ -39,7 +39,7 @@ from xdis.util import code2num # Note: these all have to be floats PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, - 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)) + 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8)) CANONIC2VERSION = dict((canonic_python_version[str(v)], v) for v in PYTHON_VERSIONS) diff --git a/uncompyle6/scanners/scanner13.py b/uncompyle6/scanners/scanner13.py index 611199dd..f605fcf9 100644 --- a/uncompyle6/scanners/scanner13.py +++ b/uncompyle6/scanners/scanner13.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 by Rocky Bernstein +# Copyright (c) 2018-2019 by Rocky Bernstein """ Python 1.3 bytecode decompiler massaging. @@ -25,7 +25,7 @@ class Scanner13(scan.Scanner14): self.version = 1.3 return - # def ingest22(self, co, classname=None, code_objects={}, show_asm=None): + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] diff --git a/uncompyle6/scanners/scanner14.py b/uncompyle6/scanners/scanner14.py index 5e80cf4a..f8703897 100644 --- a/uncompyle6/scanners/scanner14.py +++ b/uncompyle6/scanners/scanner14.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 by Rocky Bernstein +# Copyright (c) 2018-2019 by Rocky Bernstein """ Python 1.4 bytecode decompiler massaging. @@ -26,7 +26,7 @@ class Scanner14(scan.Scanner15): self.genexpr_name = '' return - # def ingest22(self, co, classname=None, code_objects={}, show_asm=None): + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 343944c6..693a6d0f 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -64,8 +64,13 @@ class Scanner3(Scanner): # Ops that start SETUP_ ... We will COME_FROM with these names # Some blocks and END_ statements. And they can start # a new statement - setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT, - self.opc.SETUP_FINALLY] + if self.version < 3.8: + setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT, + self.opc.SETUP_FINALLY] + self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP]) + else: + setup_ops = [self.opc.SETUP_FINALLY] + self.setup_ops_no_loop = frozenset(setup_ops) if self.version >= 3.2: setup_ops.append(self.opc.SETUP_WITH) @@ -78,11 +83,9 @@ class Scanner3(Scanner): self.pop_jump_tf = frozenset([self.opc.PJIF, self.opc.PJIT]) self.not_continue_follow = ('END_FINALLY', 'POP_BLOCK') - self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP]) # Opcodes that can start a statement. statement_opcodes = [ - self.opc.BREAK_LOOP, self.opc.CONTINUE_LOOP, self.opc.POP_BLOCK, self.opc.STORE_FAST, self.opc.DELETE_FAST, self.opc.STORE_DEREF, @@ -97,6 +100,9 @@ class Scanner3(Scanner): self.opc.PRINT_EXPR, self.opc.JUMP_ABSOLUTE ] + if self.version < 3.8: + statement_opcodes += [self.opc.BREAK_LOOP, self.opc.CONTINUE_LOOP] + self.statement_opcodes = frozenset(statement_opcodes) | self.setup_ops_no_loop # Opcodes that can start a "store" non-terminal. @@ -628,7 +634,7 @@ class Scanner3(Scanner): end = current_end parent = struct - if op == self.opc.SETUP_LOOP: + if self.version < 3.8 and op == self.opc.SETUP_LOOP: # We categorize loop types: 'for', 'while', 'while 1' with # possibly suffixes '-loop' and '-else' # Try to find the jump_back instruction of the loop. @@ -857,6 +863,11 @@ class Scanner3(Scanner): # if the condition jump is to a forward location. # Also the existence of a jump to the instruction after "END_FINALLY" # will distinguish "try/else" from "try". + if self.version < 3.8: + rtarget_break = (self.opc.RETURN_VALUE, self.opc.BREAK_LOOP) + else: + rtarget_break = (self.opc.RETURN_VALUE,) + if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5): if_end = self.get_target(pre_rtarget) @@ -891,8 +902,7 @@ class Scanner3(Scanner): 'start': start, 'end': pre_rtarget}) self.not_continue.add(pre_rtarget) - elif code[pre_rtarget] in (self.opc.RETURN_VALUE, - self.opc.BREAK_LOOP): + elif code[pre_rtarget] in rtarget_break: self.structs.append({'type': 'if-then', 'start': start, 'end': rtarget}) @@ -957,7 +967,7 @@ class Scanner3(Scanner): if rtarget > offset: self.fixed_jumps[offset] = rtarget - elif op == self.opc.SETUP_EXCEPT: + elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT: target = self.get_target(offset) end = self.restrict_to_parent(target, parent) self.fixed_jumps[offset] = end diff --git a/uncompyle6/scanners/scanner38.py b/uncompyle6/scanners/scanner38.py new file mode 100644 index 00000000..fb6f575e --- /dev/null +++ b/uncompyle6/scanners/scanner38.py @@ -0,0 +1,51 @@ +# Copyright (c) 2019 by Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Python 3.8 bytecode decompiler scanner + +Does some additional massaging of xdis-disassembled instructions to +make things easier for decompilation. + +This sets up opcodes Python's 3.8 and calls a generalized +scanner routine for Python 3. +""" + +from __future__ import print_function + +from uncompyle6.scanners.scanner3 import Scanner3 + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_38 as opc +JUMP_OPs = opc.JUMP_OPS + +class Scanner38(Scanner3): + + def __init__(self, show_asm=None): + Scanner3.__init__(self, 3.8, show_asm) + return + pass + +if __name__ == "__main__": + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.8: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner38().ingest(co) + for t in tokens: + print(t.format()) + pass + else: + print("Need to be Python 3.8 to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index 51b4be25..d4aed209 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017, 2018 by Rocky Bernstein +# Copyright (c) 2017-2019 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -276,13 +276,28 @@ TABLE_DIRECT = { 'while1elsestmt': ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ), 'whileelsestmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ), 'whileelselaststmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ), - 'for': ( '%|for %c in %c:\n%+%c%-\n\n', (3, 'store'), 1, 4 ), + + # Note: Python 3.8+ changes this + 'for': ( '%|for %c in %c:\n%+%c%-\n\n', + (3, 'store'), + (1, 'expr'), + (4, 'for_block') ), 'forelsestmt': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', (3, 'store'), 1, 4, -2 ), + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (3, 'store'), + (1, 'expr'), + (4, 'for_block'), -2 ), 'forelselaststmt': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', (3, 'store'), 1, 4, -2 ), + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', + (3, 'store'), + (1, 'expr'), + (4, 'for_block'), -2 ), 'forelselaststmtl': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', (3, 'store'), 1, 4, -2 ), + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (3, 'store'), + (1, 'expr'), + (4, 'for_block'), -2 ), + 'try_except': ( '%|try:\n%+%c%-%c\n\n', 1, 3 ), 'tryelsestmt': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ), 'tryelsestmtc': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ), diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index d8fa1200..6ac0441d 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -908,6 +908,10 @@ def customize_for_version3(self, version): self.n_return_closure = return_closure if version >= 3.7: + ######################## + # Python 3.7+ changes + ####################### + PRECEDENCE['attribute37'] = 2 TABLE_DIRECT.update({ 'attribute37': ( '%c.%[1]{pattr}', 0 ), @@ -928,7 +932,37 @@ def customize_for_version3(self, version): 'compare_chained2b_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)), }) + if version >= 3.8: + ######################## + # Python 3.8+ changes + ####################### + for lhs in 'for forelsestmt forelselaststmt forelselaststmtl'.split(): + del TABLE_DIRECT[lhs] + TABLE_DIRECT.update({ + 'for38': ( + '%|for %c in %c:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block') ), + 'forelsestmt38': ( + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block'), -2 ), + 'forelselaststmt38': ( + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', + (2, 'store'), + (0, 'expr'), + (3, 'for_block'), -2 ), + 'forelselaststmtl38': ( + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block'), -2 ), + }) + pass pass + pass # version >= 3.6 pass # version >= 3.4 return