From 82fb9426afa75aeab5c87901e07428bb874e5b99 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 28 Mar 2019 12:10:08 -0400 Subject: [PATCH 01/10] [WIP] - move forward a tad on Python 3.8 --- uncompyle6/parsers/parse38.py | 54 ++++++++++++++++++++++++++++++++ uncompyle6/scanner.py | 4 +-- uncompyle6/scanners/scanner3.py | 26 ++++++++++----- uncompyle6/scanners/scanner38.py | 51 ++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 uncompyle6/parsers/parse38.py create mode 100644 uncompyle6/scanners/scanner38.py diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py new file mode 100644 index 00000000..2b6ea9b8 --- /dev/null +++ b/uncompyle6/parsers/parse38.py @@ -0,0 +1,54 @@ +# Copyright (c) 2017-2019 Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +spark grammar differences over Python 3.7 for Python 3.8 +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parsers.parse37 import Python37Parser + +class Python38Parser(Python37Parser): + + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python38Parser, self).__init__(debug_parser) + self.customized = {} + + +class Python38ParserSingle(Python38Parser, PythonParserSingle): + pass + +if __name__ == '__main__': + # Check grammar + p = Python38Parser() + p.check_grammar() + from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.8: + lhs, rhs, tokens, right_recursive = p.check_sets() + from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) + opcode_set = set(s.opc.opname).union(set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split())) + remain_tokens = set(tokens) - opcode_set + import re + remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) + remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + remain_tokens = set(remain_tokens) - opcode_set + print(remain_tokens) + # print(sorted(p.rule2name.items())) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index c9318d8b..43bdf151 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, 2018 by Rocky Bernstein +# Copyright (c) 2016, 2018-2019 by Rocky Bernstein # Copyright (c) 2005 by Dan Pascu # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock @@ -39,7 +39,7 @@ from xdis.util import code2num # Note: these all have to be floats PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, - 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)) + 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8)) CANONIC2VERSION = dict((canonic_python_version[str(v)], v) for v in PYTHON_VERSIONS) diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 343944c6..693a6d0f 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -64,8 +64,13 @@ class Scanner3(Scanner): # Ops that start SETUP_ ... We will COME_FROM with these names # Some blocks and END_ statements. And they can start # a new statement - setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT, - self.opc.SETUP_FINALLY] + if self.version < 3.8: + setup_ops = [self.opc.SETUP_LOOP, self.opc.SETUP_EXCEPT, + self.opc.SETUP_FINALLY] + self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP]) + else: + setup_ops = [self.opc.SETUP_FINALLY] + self.setup_ops_no_loop = frozenset(setup_ops) if self.version >= 3.2: setup_ops.append(self.opc.SETUP_WITH) @@ -78,11 +83,9 @@ class Scanner3(Scanner): self.pop_jump_tf = frozenset([self.opc.PJIF, self.opc.PJIT]) self.not_continue_follow = ('END_FINALLY', 'POP_BLOCK') - self.setup_ops_no_loop = frozenset(setup_ops) - frozenset([self.opc.SETUP_LOOP]) # Opcodes that can start a statement. statement_opcodes = [ - self.opc.BREAK_LOOP, self.opc.CONTINUE_LOOP, self.opc.POP_BLOCK, self.opc.STORE_FAST, self.opc.DELETE_FAST, self.opc.STORE_DEREF, @@ -97,6 +100,9 @@ class Scanner3(Scanner): self.opc.PRINT_EXPR, self.opc.JUMP_ABSOLUTE ] + if self.version < 3.8: + statement_opcodes += [self.opc.BREAK_LOOP, self.opc.CONTINUE_LOOP] + self.statement_opcodes = frozenset(statement_opcodes) | self.setup_ops_no_loop # Opcodes that can start a "store" non-terminal. @@ -628,7 +634,7 @@ class Scanner3(Scanner): end = current_end parent = struct - if op == self.opc.SETUP_LOOP: + if self.version < 3.8 and op == self.opc.SETUP_LOOP: # We categorize loop types: 'for', 'while', 'while 1' with # possibly suffixes '-loop' and '-else' # Try to find the jump_back instruction of the loop. @@ -857,6 +863,11 @@ class Scanner3(Scanner): # if the condition jump is to a forward location. # Also the existence of a jump to the instruction after "END_FINALLY" # will distinguish "try/else" from "try". + if self.version < 3.8: + rtarget_break = (self.opc.RETURN_VALUE, self.opc.BREAK_LOOP) + else: + rtarget_break = (self.opc.RETURN_VALUE,) + if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5): if_end = self.get_target(pre_rtarget) @@ -891,8 +902,7 @@ class Scanner3(Scanner): 'start': start, 'end': pre_rtarget}) self.not_continue.add(pre_rtarget) - elif code[pre_rtarget] in (self.opc.RETURN_VALUE, - self.opc.BREAK_LOOP): + elif code[pre_rtarget] in rtarget_break: self.structs.append({'type': 'if-then', 'start': start, 'end': rtarget}) @@ -957,7 +967,7 @@ class Scanner3(Scanner): if rtarget > offset: self.fixed_jumps[offset] = rtarget - elif op == self.opc.SETUP_EXCEPT: + elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT: target = self.get_target(offset) end = self.restrict_to_parent(target, parent) self.fixed_jumps[offset] = end diff --git a/uncompyle6/scanners/scanner38.py b/uncompyle6/scanners/scanner38.py new file mode 100644 index 00000000..12ba67b9 --- /dev/null +++ b/uncompyle6/scanners/scanner38.py @@ -0,0 +1,51 @@ +# Copyright (c) 2019 by Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Python 3.8 bytecode decompiler scanner + +Does some additional massaging of xdis-disassembled instructions to +make things easier for decompilation. + +This sets up opcodes Python's 3.8 and calls a generalized +scanner routine for Python 3. +""" + +from __future__ import print_function + +from uncompyle6.scanners.scanner37 import Scanner3 + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_37 as opc +JUMP_OPs = opc.JUMP_OPS + +class Scanner38(Scanner3): + + def __init__(self, show_asm=None): + Scanner3.__init__(self, 3.8, show_asm) + return + pass + +if __name__ == "__main__": + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.8: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner38().ingest(co) + for t in tokens: + print(t.format()) + pass + else: + print("Need to be Python 3.8 to demo; I am %s." % + PYTHON_VERSION) From 5a6550b353a44b4a45d8be3c951c814aecc74465 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 28 Mar 2019 20:56:49 -0400 Subject: [PATCH 02/10] Administrivia: require xdis 3.9.1 or greater --- __pkginfo__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__pkginfo__.py b/__pkginfo__.py index 135da258..adcff79d 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -57,7 +57,7 @@ entry_points = { ]} ftp_url = None install_requires = ['spark-parser >= 1.8.7, < 1.9.0', - 'xdis >= 3.9.0, < 3.10.0'] + 'xdis >= 3.9.1, < 3.10.0'] license = 'GPL3' mailing_list = 'python-debugger@googlegroups.com' From 19a95be3efa20449afaf69640e5697131e4a6792 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 30 Mar 2019 00:27:48 -0400 Subject: [PATCH 03/10] WIP - more 3.8 grammar stuff --- setup.py | 4 ++-- uncompyle6/bin/uncompile.py | 4 ++-- uncompyle6/parsers/parse38.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 59953230..dc120134 100755 --- a/setup.py +++ b/setup.py @@ -4,8 +4,8 @@ import sys """Setup script for the 'uncompyle6' distribution.""" SYS_VERSION = sys.version_info[0:2] -if not ((2, 6) <= SYS_VERSION <= (3, 7)): - mess = "Python Release 2.6 .. 3.7 are supported in this code branch." +if not ((2, 6) <= SYS_VERSION <= (3, 8)): + mess = "Python Release 2.6 .. 3.8 are supported in this code branch." if ((2, 4) <= SYS_VERSION <= (2, 7)): mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." % sys.version[0:3]) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index 1151520a..c482bcbd 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -72,9 +72,9 @@ def main_bin(): if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6), - (3, 7) + (3, 7), (3, 8) )): - print('Error: %s requires Python 2.6-3.7' % program, + print('Error: %s requires Python 2.6-3.8' % program, file=sys.stderr) sys.exit(-1) diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index 2b6ea9b8..f2d34b28 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -23,10 +23,39 @@ from uncompyle6.parsers.parse37 import Python37Parser class Python38Parser(Python37Parser): + def p_38misc(self, args): + """ + for ::= expr get_iter store for_block JUMP_BACK + forelsestmt ::= expr for_iter store for_block POP_BLOCK else_suite + forelselaststmt ::= expr for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl ::= expr for_iter store for_block POP_BLOCK else_suitel + whilestmt ::= testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK + whilestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whilestmt ::= testexpr returns POP_BLOCK + while1elsestmt ::= l_stmts JUMP_BACK + whileelsestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK + while1stmt ::= l_stmts COME_FROM_LOOP + while1stmt ::= l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + while1elsestmt ::= l_stmts JUMP_BACK + whileTruestmt ::= l_stmts_opt JUMP_BACK NOP + whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK NOP + """ + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python38Parser, self).__init__(debug_parser) self.customized = {} + def customize_grammar_rules(self, tokens, customize): + from trepan.api import debug; debug() + self.remove_rules(""" + for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK + forelsestmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite + forelselaststmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel + for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP + """) + super(Python37Parser, self).customize_grammar_rules(tokens, customize) class Python38ParserSingle(Python38Parser, PythonParserSingle): pass From 1e47f475279e0ce737d8bdfa0ea574317a1fc5cc Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 5 Apr 2019 16:30:28 -0400 Subject: [PATCH 04/10] Allow for newer xdis --- __pkginfo__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/__pkginfo__.py b/__pkginfo__.py index adcff79d..33882707 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -23,7 +23,7 @@ # Things that change more often go here. copyright = """ -Copyright (C) 2015-2018 Rocky Bernstein . +Copyright (C) 2015-2019 Rocky Bernstein . """ classifiers = ['Development Status :: 5 - Production/Stable', @@ -57,7 +57,7 @@ entry_points = { ]} ftp_url = None install_requires = ['spark-parser >= 1.8.7, < 1.9.0', - 'xdis >= 3.9.1, < 3.10.0'] + 'xdis >= 3.9.1, < 4.1.0'] license = 'GPL3' mailing_list = 'python-debugger@googlegroups.com' From 8b65cc7275605688e6abce7122e377094b7e7ace Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 9 Apr 2019 21:45:28 -0400 Subject: [PATCH 05/10] Small changes - bump required xdis version --- __pkginfo__.py | 2 +- uncompyle6/scanners/scanner13.py | 4 ++-- uncompyle6/scanners/scanner14.py | 4 ++-- uncompyle6/scanners/scanner38.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/__pkginfo__.py b/__pkginfo__.py index 33882707..39fccce6 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -57,7 +57,7 @@ entry_points = { ]} ftp_url = None install_requires = ['spark-parser >= 1.8.7, < 1.9.0', - 'xdis >= 3.9.1, < 4.1.0'] + 'xdis >= 4.0.0, < 4.1.0'] license = 'GPL3' mailing_list = 'python-debugger@googlegroups.com' diff --git a/uncompyle6/scanners/scanner13.py b/uncompyle6/scanners/scanner13.py index 611199dd..f605fcf9 100644 --- a/uncompyle6/scanners/scanner13.py +++ b/uncompyle6/scanners/scanner13.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 by Rocky Bernstein +# Copyright (c) 2018-2019 by Rocky Bernstein """ Python 1.3 bytecode decompiler massaging. @@ -25,7 +25,7 @@ class Scanner13(scan.Scanner14): self.version = 1.3 return - # def ingest22(self, co, classname=None, code_objects={}, show_asm=None): + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] diff --git a/uncompyle6/scanners/scanner14.py b/uncompyle6/scanners/scanner14.py index 5e80cf4a..f8703897 100644 --- a/uncompyle6/scanners/scanner14.py +++ b/uncompyle6/scanners/scanner14.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018 by Rocky Bernstein +# Copyright (c) 2018-2019 by Rocky Bernstein """ Python 1.4 bytecode decompiler massaging. @@ -26,7 +26,7 @@ class Scanner14(scan.Scanner15): self.genexpr_name = '' return - # def ingest22(self, co, classname=None, code_objects={}, show_asm=None): + # def ingest(self, co, classname=None, code_objects={}, show_asm=None): # tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm) # tokens = [t for t in tokens if t.kind != 'SET_LINENO'] diff --git a/uncompyle6/scanners/scanner38.py b/uncompyle6/scanners/scanner38.py index 12ba67b9..fb6f575e 100644 --- a/uncompyle6/scanners/scanner38.py +++ b/uncompyle6/scanners/scanner38.py @@ -24,10 +24,10 @@ scanner routine for Python 3. from __future__ import print_function -from uncompyle6.scanners.scanner37 import Scanner3 +from uncompyle6.scanners.scanner3 import Scanner3 # bytecode verification, verify(), uses JUMP_OPs from here -from xdis.opcodes import opcode_37 as opc +from xdis.opcodes import opcode_38 as opc JUMP_OPs = opc.JUMP_OPS class Scanner38(Scanner3): From 820283827f46142ce341fb0d5e1d4e0ab2c9ff47 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 10 Apr 2019 06:00:16 -0400 Subject: [PATCH 06/10] 3.8 "for" block ... pysource: Tag older semantics for blocks with "expr" and "for_block" --- uncompyle6/parser.py | 6 ++++++ uncompyle6/parsers/parse38.py | 7 +++++-- uncompyle6/semantics/consts.py | 20 ++++++++++++++++---- uncompyle6/semantics/customize3.py | 16 ++++++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 08d701c0..e7e5675c 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -747,6 +747,12 @@ def get_python_parser( p = parse37.Python37Parser(debug_parser) else: p = parse37.Python37ParserSingle(debug_parser) + elif version == 3.8: + import uncompyle6.parsers.parse38 as parse38 + if compile_mode == 'exec': + p = parse38.Python38Parser(debug_parser) + else: + p = parse38.Python38ParserSingle(debug_parser) else: if compile_mode == 'exec': p = parse3.Python3Parser(debug_parser) diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index f2d34b28..fb0d5e6e 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -25,7 +25,10 @@ class Python38Parser(Python37Parser): def p_38misc(self, args): """ - for ::= expr get_iter store for_block JUMP_BACK + stmt ::= for38 + for38 ::= expr get_iter store for_block JUMP_BACK + for38 ::= expr for_iter store for_block JUMP_BACK + forelsestmt ::= expr for_iter store for_block POP_BLOCK else_suite forelselaststmt ::= expr for_iter store for_block POP_BLOCK else_suitec forelselaststmtl ::= expr for_iter store for_block POP_BLOCK else_suitel @@ -40,6 +43,7 @@ class Python38Parser(Python37Parser): while1elsestmt ::= l_stmts JUMP_BACK whileTruestmt ::= l_stmts_opt JUMP_BACK NOP whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK NOP + for_block ::= l_stmts_opt _come_from_loops JUMP_BACK """ def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): @@ -47,7 +51,6 @@ class Python38Parser(Python37Parser): self.customized = {} def customize_grammar_rules(self, tokens, customize): - from trepan.api import debug; debug() self.remove_rules(""" for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK forelsestmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index 51b4be25..f68580c2 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -276,13 +276,25 @@ TABLE_DIRECT = { 'while1elsestmt': ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ), 'whileelsestmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ), 'whileelselaststmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ), - 'for': ( '%|for %c in %c:\n%+%c%-\n\n', (3, 'store'), 1, 4 ), + 'for': ( '%|for %c in %c:\n%+%c%-\n\n', + (3, 'store'), + (1, 'expr'), + (4, 'for_block') ), 'forelsestmt': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', (3, 'store'), 1, 4, -2 ), + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (3, 'store'), + (1, 'expr'), + (4, 'for_block'), -2 ), 'forelselaststmt': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', (3, 'store'), 1, 4, -2 ), + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', + (3, 'store'), + (1, 'expr'), + (4, 'for_block'), -2 ), 'forelselaststmtl': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', (3, 'store'), 1, 4, -2 ), + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (3, 'store'), + (1, 'expr'), + (4, 'for_block'), -2 ), 'try_except': ( '%|try:\n%+%c%-%c\n\n', 1, 3 ), 'tryelsestmt': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ), 'tryelsestmtc': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ), diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index d8fa1200..326f0beb 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -908,6 +908,10 @@ def customize_for_version3(self, version): self.n_return_closure = return_closure if version >= 3.7: + ######################## + # Python 3.7+ Additions + ####################### + PRECEDENCE['attribute37'] = 2 TABLE_DIRECT.update({ 'attribute37': ( '%c.%[1]{pattr}', 0 ), @@ -928,7 +932,19 @@ def customize_for_version3(self, version): 'compare_chained2b_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)), }) + if version >= 3.8: + ######################## + # Python 3.8+ Additions + ####################### + TABLE_DIRECT.update({ + 'for38': ( '%|for %c in %c:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block') ), + }) + pass pass + pass # version >= 3.6 pass # version >= 3.4 return From f3d86e070817068293ab5d761338a5d086dc88b7 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 10 Apr 2019 07:22:43 -0400 Subject: [PATCH 07/10] Bang on Python 3.8 --- test/Makefile | 38 ++++++++++++++++----------- test/add-test.py | 13 ++++++--- test/bytecode_3.3/10_for.pyc | Bin 221 -> 0 bytes test/bytecode_3.3_run/10_for.pyc | Bin 0 -> 316 bytes test/bytecode_3.4/10_for.pyc | Bin 195 -> 0 bytes test/bytecode_3.4_run/10_for.pyc | Bin 0 -> 289 bytes test/bytecode_3.5/10_for.pyc | Bin 197 -> 0 bytes test/bytecode_3.5_run/10_for.pyc | Bin 0 -> 289 bytes test/bytecode_3.8/09_yield_from.pyc | Bin 0 -> 466 bytes test/bytecode_3.8_run/10_for.pyc | Bin 0 -> 259 bytes test/simple_source/looping/10_for.py | 13 ++++++--- test/test_pythonlib.py | 2 +- uncompyle6/parsers/parse3.py | 4 ++- uncompyle6/parsers/parse36.py | 15 +++++++++-- uncompyle6/parsers/parse38.py | 2 ++ 15 files changed, 60 insertions(+), 27 deletions(-) delete mode 100644 test/bytecode_3.3/10_for.pyc create mode 100644 test/bytecode_3.3_run/10_for.pyc delete mode 100644 test/bytecode_3.4/10_for.pyc create mode 100644 test/bytecode_3.4_run/10_for.pyc delete mode 100644 test/bytecode_3.5/10_for.pyc create mode 100644 test/bytecode_3.5_run/10_for.pyc create mode 100644 test/bytecode_3.8/09_yield_from.pyc create mode 100644 test/bytecode_3.8_run/10_for.pyc diff --git a/test/Makefile b/test/Makefile index 6d12cf0a..f948cb03 100644 --- a/test/Makefile +++ b/test/Makefile @@ -33,23 +33,23 @@ check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-na #: Run working tests from Python 3.0 check-3.0: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE) #: Run working tests from Python 3.1 check-3.1: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE) #: Run working tests from Python 3.2 check-3.2: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(COMPILE) #: Run working tests from Python 3.3 check-3.3: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE) #: Run working tests from Python 3.4 check-3.4: check-bytecode check-3.4-ok check-2.7-ok @@ -58,18 +58,23 @@ check-3.4: check-bytecode check-3.4-ok check-2.7-ok #: Run working tests from Python 3.5 check-3.5: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(COMPILE) #: Run working tests from Python 3.6 check-3.6: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE) #: Run working tests from Python 3.7 check-3.7: check-bytecode - $(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE) + +#: Run working tests from Python 3.8 +check-3.8: check-bytecode + $(PYTHON) test_pythonlib.py --bytecode-3.8-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify $(COMPILE) # FIXME #: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0 @@ -92,7 +97,8 @@ check-bytecode-2: check-bytecode-3: $(PYTHON) test_pythonlib.py --bytecode-3.0 \ --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \ - --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-3.7 \ + --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \ + --bytecode-3.7 --bytecode-3.8 \ --bytecode-pypy3.2 #: Check deparsing on selected bytecode 3.x @@ -217,43 +223,43 @@ grammar-coverage-3.6: #: Check deparsing Python 2.6 check-bytecode-2.6: - $(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify #: Check deparsing Python 2.7 check-bytecode-2.7: - $(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify #: Check deparsing Python 3.0 check-bytecode-3.0: - $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify #: Check deparsing Python 3.1 check-bytecode-3.1: - $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify #: Check deparsing Python 3.2 check-bytecode-3.2: - $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.2-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.2 --weak-verify #: Check deparsing Python 3.3 check-bytecode-3.3: - $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify #: Check deparsing Python 3.4 check-bytecode-3.4: - $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify #: Check deparsing Python 3.5 check-bytecode-3.5: - $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify $(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify #: Check deparsing Python 3.6 check-bytecode-3.6: diff --git a/test/add-test.py b/test/add-test.py index 905323d5..a5ea9ac6 100755 --- a/test/add-test.py +++ b/test/add-test.py @@ -4,12 +4,19 @@ import os, sys, py_compile assert len(sys.argv) >= 2 version = sys.version[0:3] -for path in sys.argv[1:]: +if sys.argv[1] == '--run': + suffix = '_run' + py_source = sys.argv[2:] +else: + suffix = '' + py_source = sys.argv[1:] + +for path in py_source: short = os.path.basename(path) if hasattr(sys, 'pypy_version_info'): - cfile = "bytecode_pypy%s/%s" % (version, short) + 'c' + cfile = "bytecode_pypy%s%s/%s" % (version, suffix, short) + 'c' else: - cfile = "bytecode_%s/%s" % (version, short) + 'c' + cfile = "bytecode_%s%s/%s" % (version, suffix, short) + 'c' print("byte-compiling %s to %s" % (path, cfile)) py_compile.compile(path, cfile) if isinstance(version, str) or version >= (2, 6, 0): diff --git a/test/bytecode_3.3/10_for.pyc b/test/bytecode_3.3/10_for.pyc deleted file mode 100644 index 3379248ff9c15db21f33e7954d3f5fef7532f4d1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 221 zcmbQo!^_neRT(y$fq@|zh~a<<$Z`PUVrw8#AvK1v#nl#rdU0$*KA|`S}HzdFlFw2Jvb6MS2C5r5r$2Ho5sJr8%i~EI@WK N8;}rS`LhQx$6IjJKH@6vkA#%m=r_<=WVOwf9Xv XGujyG3pEZGxi+Ig3B}YE9g4*dy!tma literal 0 HcmV?d00001 diff --git a/test/bytecode_3.4/10_for.pyc b/test/bytecode_3.4/10_for.pyc deleted file mode 100644 index 26c63996bc670690971dec026b6c7e887cfd7591..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmaFI!^@RWUlBH&fq~&M5W@izkmUfx#nwQgLYyImfgzod!6lZ5A&P+^g^?kOk)e=_ zA-qD0A(aWlZ$_6?X9!PWVhGk`e#r$fh-3gF0KR;DGUrz3=9?03@MBZ>5L36u>uTHj0~v^ z45^F^=0KT30fulO4=9_>#87O*P{IU~Y-VI=Vq~b0V@PFYNMQnTF*SNZG%|xVmZE4( zVPOc?WPJ&=p5Y}UhyXe1B@2-7(`32Dn0$*d@fM$BadB! zl?+9kKxJU!mwa(%Zb43Jd~tqhQF5w&PJVttW?s6!p+S6Fevw{55cz{tla!6eMc#{>W)zdft~ literal 0 HcmV?d00001 diff --git a/test/bytecode_3.5/10_for.pyc b/test/bytecode_3.5/10_for.pyc deleted file mode 100644 index 82d6433788f293db0d19328404b5275432685eee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 197 zcmWgR<>gAKuLzsXz`*brh~a<<$Z`PUVrw8#Av88U$+uXG67$kiS27f_0foWD zFGW55;>_HFoYeT@{L-T2RQ;U%{DRE9bbUjE__X{Yy@JYH95%W6DWy57b}T?$#UQN$ Jj9g5@i~x4rDGLAq diff --git a/test/bytecode_3.5_run/10_for.pyc b/test/bytecode_3.5_run/10_for.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e669db77acb447129cc5c13868fc893dd79f6dd0 GIT binary patch literal 289 zcmWgV<>k6|W^K$N1_p-5Knw@WK$Zg#7gqv_6b6PU28If0h7?AIbVdf3SOJD8Mut=d zhEzrdbD&J207E#C2b9fbVkov@C}9FgHZw9bF)~!hF{Cmxq%eWFm>NAH8kxZwOHnkY zurLH`vc3db&+w8FM1Y+1k_AZkX|mj6OuogKc#F@mxHz?_Br`wHwWuh+=oV{HVqSXc zN`@j%pfWJ=OTIWWw;(4qzBs?MC^=O>CqKU+GcR4=&>%i7zeum3@)n0pZhlH>PO2R% WP){)rkYHmJVB}+zU=n8JV*&s&89dei literal 0 HcmV?d00001 diff --git a/test/bytecode_3.8/09_yield_from.pyc b/test/bytecode_3.8/09_yield_from.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8da7c6ad8cd768d4caa9fcb989a4c2878acf3fff GIT binary patch literal 466 zcmZ8eO-lnY5S>Z3wI5VGi1-I8?8SN!6hu5I+DkzNEejHsoh+=~b!&Eu7PhDMDjxh5 z{-wS8FFZL((Ss8v^Ipg#?f0DM{U8x zTkxBs$ZoS?tH&8}x|R|usF3hYTmZopAkW+)Mn2XaQDV5Z)k~O8lqIs3T4RMLsi;9_ZJp(M6HuPlJPFHBWR(DmHnxKb;%)09`A<0zk(jhnZS+ zZBD1~h(U!FEb)C0S?MF3!^7@ERALDWSxJIP+iX?xj&(n8-NPYV!wrnr9&qg;EJNR` zy+WsPusJNX2}@RWISUO18%e1%qvP6hz#4IGOp{UFQi_+2Qub{N_XpVF(Zm)SOMSjV WIQfpQTmK(&lI1i@_35AEPWJ~*L0ljJ literal 0 HcmV?d00001 diff --git a/test/bytecode_3.8_run/10_for.pyc b/test/bytecode_3.8_run/10_for.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38a345ee98bf7862af676210aad21fdaf11ccb10 GIT binary patch literal 259 zcmea|<>g`kf@^2i#vB6Dk3k$5Ugy`3o`T4^$iW;)AEb-3My}L*yQG?l;)(`u>zGB^8g7JMh-?UMgb-s HMlL1*2$njy literal 0 HcmV?d00001 diff --git a/test/simple_source/looping/10_for.py b/test/simple_source/looping/10_for.py index eb85f549..3a37c4cf 100644 --- a/test/simple_source/looping/10_for.py +++ b/test/simple_source/looping/10_for.py @@ -1,8 +1,13 @@ # Tests: # forstmt ::= SETUP_LOOP expr _for store # for_block POP_BLOCK COME_FROM -for a in [1]: - c = 2 -for a in range(2): - c = 2 +c = 0 +for a in [1]: + c += a +assert c == 1, c + +for a in range(3): + c += a + +assert c == 4, c diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index e74d9d46..df05fcf5 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -81,7 +81,7 @@ for vers in (2.7, 3.4, 3.5, 3.6): for vers in (1.3, 1.4, 1.5, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.0, 3.1, 3.2, 3.3, - 3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'): + 3.4, 3.5, 3.6, 3.7, 3.8, 'pypy3.2', 'pypy2.7'): bytecode = "bytecode_%s" % vers key = "bytecode-%s" % vers test_options[key] = (bytecode, PYC, bytecode, vers) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index eacabdcd..98321047 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1190,7 +1190,9 @@ class Python3Parser(PythonParser): last += 1 if last == n: return False - return tokens[first].attr > tokens[last].offset + # 3.8+ Doesn't have SETUP_LOOP + return self.version < 3.8 and tokens[first].attr > tokens[last].offset + elif rule == ('try_except', ('SETUP_EXCEPT', 'suite_stmts_opt', 'POP_BLOCK', 'except_handler', 'opt_come_from_except')): diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 84a696b0..05ab5829 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -231,8 +231,6 @@ class Python36Parser(Python35Parser): self.addRule(rule, nop_func) elif opname == 'SETUP_WITH': rules_str = """ - withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST - WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY @@ -240,6 +238,19 @@ class Python36Parser(Python35Parser): withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY """ + if self.version < 3.8: + rules_str += """ + withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + LOAD_CONST + WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY + """ + else: + rules_str += """ + withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK + BEGIN_FINALLY COME_FROM_WITH + WITH_CLEANUP_START WITH_CLEANUP_FINISH + END_FINALLY + """ self.addRule(rules_str, nop_func) pass pass diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index fb0d5e6e..014d1cc5 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -26,8 +26,10 @@ class Python38Parser(Python37Parser): def p_38misc(self, args): """ stmt ::= for38 + for38 ::= expr get_iter store for_block JUMP_BACK for38 ::= expr for_iter store for_block JUMP_BACK + for38 ::= expr for_iter store for_block JUMP_BACK POP_BLOCK forelsestmt ::= expr for_iter store for_block POP_BLOCK else_suite forelselaststmt ::= expr for_iter store for_block POP_BLOCK else_suitec From 49e354375e77fb6b543a98dc76b27a1a7851ac1f Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 10 Apr 2019 11:05:46 -0400 Subject: [PATCH 08/10] More run tests --- test/Makefile | 2 +- test/bytecode_3.3/10_mixed_boolean.pyc | Bin 411 -> 0 bytes test/bytecode_3.3_run/10_mixed_boolean.pyc | Bin 0 -> 717 bytes test/bytecode_3.4/10_mixed_boolean.pyc | Bin 417 -> 0 bytes test/bytecode_3.4_run/10_mixed_boolean.pyc | Bin 0 -> 650 bytes test/bytecode_3.5/10_mixed_boolean.pyc | Bin 365 -> 0 bytes test/bytecode_3.5_run/10_mixed_boolean.pyc | Bin 0 -> 650 bytes test/simple_source/bug36/00_if_elif.py | 1 + test/simple_source/bug36/01_fstring.py | 3 ++- test/simple_source/looping/10_for.py | 8 ++++++-- .../operation_logic/10_mixed_boolean.py | 12 ++++++++++++ 11 files changed, 22 insertions(+), 4 deletions(-) delete mode 100644 test/bytecode_3.3/10_mixed_boolean.pyc create mode 100644 test/bytecode_3.3_run/10_mixed_boolean.pyc delete mode 100644 test/bytecode_3.4/10_mixed_boolean.pyc create mode 100644 test/bytecode_3.4_run/10_mixed_boolean.pyc delete mode 100644 test/bytecode_3.5/10_mixed_boolean.pyc create mode 100644 test/bytecode_3.5_run/10_mixed_boolean.pyc diff --git a/test/Makefile b/test/Makefile index f948cb03..e5111541 100644 --- a/test/Makefile +++ b/test/Makefile @@ -53,8 +53,8 @@ check-3.3: check-bytecode #: Run working tests from Python 3.4 check-3.4: check-bytecode check-3.4-ok check-2.7-ok - $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE) $(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE) #: Run working tests from Python 3.5 check-3.5: check-bytecode diff --git a/test/bytecode_3.3/10_mixed_boolean.pyc b/test/bytecode_3.3/10_mixed_boolean.pyc deleted file mode 100644 index ecd44155ab2890b9c3832e2e54a634dda0d3cc9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 411 zcmY+Ay-EZz5P&D{?jfR$g;)u8Tiu>qD`Ml7HXa^qv+QjKS;*dm#DIsDt>7zI2)=}s zU}xboIT_tsE+JngUuMY9>;A#sQ@OtV0f2_brX6u~$-{p$w1QHB-hncaI0aPTP5=|I zr+_JxSu}nU4NV7QuOx@r=g>lOn7y&B7-nziQL31|anE922IJqP>OC4_@(nY&!)NsF zBYMwq2K4p6s{wBEgr9RgpXDQ-L&})DfI)Gv3>kA)8~?X%#A;!k^A-#UQ(le>-*(nu z-i6@Kiyp1;LAoBIfAD9;AvT%`T;qcEfHL^d#urNATiN z@ZeMU2HrjR2Hu_7wn=Q*pV|4p{WsIR__gKl^85801BjvUTgUnsm!6`-E6`&g80aQU zOTYo17(6oIRp>b&9Po9=8f;8xMtRmc8t-7u?+m$*%0;9|-=0VBs%V z33e8Kz_#|bij|Y-dYEAz!^;dz^1iyZ^7&DpzXHG;=id=r9dWuBMglNl3Ydsc_W%Lx z7N7vL4cLQ8-y1jbhW6&h9EToe*FzVfhuImUgJE`tuEP|wGxjE!+qrRfVd~x+V)BMR zJ3M)BAH8>6_ko^&Z<8LJ4f^Ag)4@V3rmaw|NLmp;fL-J_27IN`StYS9(H3zwyi!i4 zx|~&0X-Qo+(lq6aieXhK%1x8uzAVv|(&^k}7&)q}LP-W~Txu$!j8d)S?kLK2jY%{m d$}lc=Ow%qO2tWX_rnCoGgg7Ah@U~Vn-J{NEJhp_b8FHFTB@+W)XDvOE`(( z;HU5#2pJU|-CUf!_u3{g;r`s^|GP^r$+ulw{3yL#JQILVC>#dn2N-gU4h84~Fszz@ zH6?IQ;0_Z5`UWrqY}wI9X>MRcNi~QgWyf7$CFq;LOpxcAG-sq5I#3M1DloU2Dd$Z5 zpy!}wa~?HgkVEmihEV5_{89{VgO9?0^leZx*Cd%MR)|`rL2`F3 zcjdNn`Hpgx;wzXg+$}JdiVT-jJiJO5%Wfy@l9IU9dbH}*uMIA Yck=qjQYQ`ft8&_6Z2K_2QzMoqyPW_ literal 0 HcmV?d00001 diff --git a/test/bytecode_3.5/10_mixed_boolean.pyc b/test/bytecode_3.5/10_mixed_boolean.pyc deleted file mode 100644 index d8be2198289a5faf228e096d60c4b41f5fe34942..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 365 zcmWgR<>mVEy&~)!0|UcjAcg}*Aj<)Wi%$WG6b6PUAj!xO#Rz6IF{Cmu6v;89GBOk> zGNdvwL@|RziVZ*_`Njy50#~paAPG~G9}ZRnBw->2nP3qh2@}b$2I~TnFp;7zut+aL zq<9fn*HVy3K9B^u5aNadAn~GIAn{@#o0%d1FjO{$i6K~%ImFFRljRm;(k;g1TZ}2U z7!y}A6tM#R2qu0R>**I~<`(3n#uw+87A2?Z=NF_FC6;97=f&sbr)MVX8ydvtW>%!8 o#3$wF=cFd)=@nGo;;_lhPbtkwwF3oOu^5nGV+27V6ez(60GfPE?EnA( diff --git a/test/bytecode_3.5_run/10_mixed_boolean.pyc b/test/bytecode_3.5_run/10_mixed_boolean.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76f642988c3f9f03e257fd1c2a81064fc91058c5 GIT binary patch literal 650 zcmY+C!AiqG5Qb-yw6@xcvoB$!fj6_-Vdgz7T*fm^%#24>9Bz9SYC~U|6*P zYf9jOz&$1g^bKGJ)MQs1W@cc6sT#zwvSSCB34IHg1@c0Z){Im`2a4g>bIhG)N(B=? z>N%+SoF~nc3x>bxIj9*^lkXZ@D4_U5L#P=fzZHYq;G^&#eH+x=wMg!Y6{41DlEPgt zT)CrMzN=iN_zI>AJ9EsHV#K*(B<2=yZ;p&-B(1<%yV^@CzP)ev#DqCXHNe#IBFU1P zZ+jzZ2R0WrO_n!&l>}{lktR#ia%$EyR5j4kb}1!(x_l%h6XTGNPK4khB|%DPO1x(f zW&Gk=e!DUp#u0NRpNN2Uc+7;STHJ^a!l2XLc8B2&qpr_+#Jtf~e4AGGhMZ0!cA)+p YxrtIH4N@jejK;sSiB&C+7UpZ@4-@l<&Hw-a literal 0 HcmV?d00001 diff --git a/test/simple_source/bug36/00_if_elif.py b/test/simple_source/bug36/00_if_elif.py index 0867a01f..747ec4bc 100644 --- a/test/simple_source/bug36/00_if_elif.py +++ b/test/simple_source/bug36/00_if_elif.py @@ -1,3 +1,4 @@ +# Self-checking test. # Python 3 bug in not detecting the end bounds of if elif. def testit(b): if b == 1: diff --git a/test/simple_source/bug36/01_fstring.py b/test/simple_source/bug36/01_fstring.py index 26bc3904..c03aaa79 100644 --- a/test/simple_source/bug36/01_fstring.py +++ b/test/simple_source/bug36/01_fstring.py @@ -1,4 +1,5 @@ -# Self-checking 3.6+ string interpolation tests +# Self-checking test. +# String interpolation tests var1 = 'x' var2 = 'y' diff --git a/test/simple_source/looping/10_for.py b/test/simple_source/looping/10_for.py index 3a37c4cf..5de22cab 100644 --- a/test/simple_source/looping/10_for.py +++ b/test/simple_source/looping/10_for.py @@ -1,6 +1,10 @@ +# Self-checking test. # Tests: -# forstmt ::= SETUP_LOOP expr _for store -# for_block POP_BLOCK COME_FROM +# for ::= SETUP_LOOP expr for_iter store +# for_block POP_BLOCK COME_FROM +# In 3.8+ +# for ::= expr for_iter store +# for_block POP_BLOCK COME_FROM c = 0 for a in [1]: diff --git a/test/simple_source/operation_logic/10_mixed_boolean.py b/test/simple_source/operation_logic/10_mixed_boolean.py index d0851a63..aafd2c80 100644 --- a/test/simple_source/operation_logic/10_mixed_boolean.py +++ b/test/simple_source/operation_logic/10_mixed_boolean.py @@ -1,12 +1,24 @@ +# Self-checking test. b = True +assert b, 'b = True' c = False +assert not c, 'c = False' d = True a = b and c or d +assert a, 'b and c or d' a = (b or c) and d +assert a, '(b or c) and d' a = b or c or d +assert a, 'b or c or d' a = b and c and d +assert not a, 'b and c and d' a = b or c and d +assert a a = b and (c or d) +assert a a = b and c or d +assert a a = (b or c and d) and b +assert a a = (b or c and d or a) and b +assert a From 726045a05e594bd7cd2a7f6c0a9392f9b4153389 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 10 Apr 2019 11:26:58 -0400 Subject: [PATCH 09/10] Basic 3.8+ "for" loop handling... More Makefile mangling --- Makefile | 3 ++ test/Makefile | 4 +++ uncompyle6/parsers/parse38.py | 55 +++++++++++++++++------------- uncompyle6/semantics/consts.py | 5 ++- uncompyle6/semantics/customize3.py | 30 ++++++++++++---- 5 files changed, 66 insertions(+), 31 deletions(-) diff --git a/Makefile b/Makefile index 5fe5d565..21d91675 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,9 @@ check-3.0 check-3.1 check-3.2 check-3.6: check-3.7: pytest $(MAKE) -C test check +check-3.8: + $(MAKE) -C test check + #:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0 # Skip for now 2.6 5.0 5.3 5.6 5.8: diff --git a/test/Makefile b/test/Makefile index e5111541..91c0a7e4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -270,6 +270,10 @@ check-bytecode-3.6: check-bytecode-3.7: $(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify +#: Check deparsing Python 3.8 +check-bytecode-3.8: + $(PYTHON) test_pythonlib.py --bytecode-3.8 --weak-verify + #: short tests for bytecodes only for this version of Python check-native-short: $(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE) diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index 014d1cc5..5594f2c8 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -25,27 +25,30 @@ class Python38Parser(Python37Parser): def p_38misc(self, args): """ - stmt ::= for38 + stmt ::= for38 + stmt ::= forelsestmt38 + stmt ::= forelselaststmt38 + stmt ::= forelselaststmtl38 - for38 ::= expr get_iter store for_block JUMP_BACK - for38 ::= expr for_iter store for_block JUMP_BACK - for38 ::= expr for_iter store for_block JUMP_BACK POP_BLOCK + for38 ::= expr get_iter store for_block JUMP_BACK + for38 ::= expr for_iter store for_block JUMP_BACK + for38 ::= expr for_iter store for_block JUMP_BACK POP_BLOCK - forelsestmt ::= expr for_iter store for_block POP_BLOCK else_suite - forelselaststmt ::= expr for_iter store for_block POP_BLOCK else_suitec - forelselaststmtl ::= expr for_iter store for_block POP_BLOCK else_suitel - whilestmt ::= testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK - whilestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK - whilestmt ::= testexpr returns POP_BLOCK - while1elsestmt ::= l_stmts JUMP_BACK - whileelsestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK - whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK - while1stmt ::= l_stmts COME_FROM_LOOP - while1stmt ::= l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP - while1elsestmt ::= l_stmts JUMP_BACK - whileTruestmt ::= l_stmts_opt JUMP_BACK NOP - whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK NOP - for_block ::= l_stmts_opt _come_from_loops JUMP_BACK + forelsestmt38 ::= expr for_iter store for_block POP_BLOCK else_suite + forelselaststmt38 ::= expr for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl38 ::= expr for_iter store for_block POP_BLOCK else_suitel + whilestmt ::= testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK + whilestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whilestmt ::= testexpr returns POP_BLOCK + while1elsestmt ::= l_stmts JUMP_BACK + whileelsestmt ::= testexpr l_stmts_opt JUMP_BACK POP_BLOCK + whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK + while1stmt ::= l_stmts COME_FROM_LOOP + while1stmt ::= l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + while1elsestmt ::= l_stmts JUMP_BACK + whileTruestmt ::= l_stmts_opt JUMP_BACK NOP + whileTruestmt ::= l_stmts_opt JUMP_BACK POP_BLOCK NOP + for_block ::= l_stmts_opt _come_from_loops JUMP_BACK """ def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): @@ -54,11 +57,15 @@ class Python38Parser(Python37Parser): def customize_grammar_rules(self, tokens, customize): self.remove_rules(""" - for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK - forelsestmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite - forelselaststmt ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec - forelselaststmtl ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel - for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP + stmt ::= for + stmt ::= forelsestmt + for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK + for ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK NOP + for_block ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK + forelsestmt38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite + forelselaststmt38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec + forelselaststmtl38 ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel + """) super(Python37Parser, self).customize_grammar_rules(tokens, customize) diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index f68580c2..d4aed209 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017, 2018 by Rocky Bernstein +# Copyright (c) 2017-2019 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -276,6 +276,8 @@ TABLE_DIRECT = { 'while1elsestmt': ( '%|while 1:\n%+%c%-%|else:\n%+%c%-\n\n', 1, -2 ), 'whileelsestmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-\n\n', 1, 2, -2 ), 'whileelselaststmt': ( '%|while %c:\n%+%c%-%|else:\n%+%c%-', 1, 2, -2 ), + + # Note: Python 3.8+ changes this 'for': ( '%|for %c in %c:\n%+%c%-\n\n', (3, 'store'), (1, 'expr'), @@ -295,6 +297,7 @@ TABLE_DIRECT = { (3, 'store'), (1, 'expr'), (4, 'for_block'), -2 ), + 'try_except': ( '%|try:\n%+%c%-%c\n\n', 1, 3 ), 'tryelsestmt': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-\n\n', 1, 3, 4 ), 'tryelsestmtc': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', 1, 3, 4 ), diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index 326f0beb..6ac0441d 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -909,7 +909,7 @@ def customize_for_version3(self, version): if version >= 3.7: ######################## - # Python 3.7+ Additions + # Python 3.7+ changes ####################### PRECEDENCE['attribute37'] = 2 @@ -934,13 +934,31 @@ def customize_for_version3(self, version): }) if version >= 3.8: ######################## - # Python 3.8+ Additions + # Python 3.8+ changes ####################### + for lhs in 'for forelsestmt forelselaststmt forelselaststmtl'.split(): + del TABLE_DIRECT[lhs] TABLE_DIRECT.update({ - 'for38': ( '%|for %c in %c:\n%+%c%-\n\n', - (2, 'store'), - (0, 'expr'), - (3, 'for_block') ), + 'for38': ( + '%|for %c in %c:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block') ), + 'forelsestmt38': ( + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block'), -2 ), + 'forelselaststmt38': ( + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', + (2, 'store'), + (0, 'expr'), + (3, 'for_block'), -2 ), + 'forelselaststmtl38': ( + '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', + (2, 'store'), + (0, 'expr'), + (3, 'for_block'), -2 ), }) pass pass From 59c77f103dbb692efbfcd28bef16299ba17866f1 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 10 Apr 2019 11:49:27 -0400 Subject: [PATCH 10/10] More self-checking run tests --- test/bytecode_2.7_run/02_slice.pyc | Bin 0 -> 323 bytes test/bytecode_3.2/02_slice.pyc | Bin 227 -> 0 bytes test/bytecode_3.2_run/02_slice.pyc | Bin 0 -> 339 bytes test/bytecode_3.3/02_slice.pyc | Bin 198 -> 0 bytes test/bytecode_3.3_run/02_slice.pyc | Bin 0 -> 343 bytes test/bytecode_3.4/02_slice.pyc | Bin 232 -> 0 bytes test/bytecode_3.4_run/02_slice.pyc | Bin 0 -> 319 bytes test/bytecode_3.5/02_slice.pyc | Bin 213 -> 0 bytes test/bytecode_3.5_run/02_slice.pyc | Bin 0 -> 319 bytes test/bytecode_3.6_run/02_slice.pyc | Bin 0 -> 274 bytes .../operation_logic/10_mixed_boolean.py | 2 ++ test/simple_source/slice/02_slice.py | 13 ++++++++++--- 12 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 test/bytecode_2.7_run/02_slice.pyc delete mode 100644 test/bytecode_3.2/02_slice.pyc create mode 100644 test/bytecode_3.2_run/02_slice.pyc delete mode 100644 test/bytecode_3.3/02_slice.pyc create mode 100644 test/bytecode_3.3_run/02_slice.pyc delete mode 100644 test/bytecode_3.4/02_slice.pyc create mode 100644 test/bytecode_3.4_run/02_slice.pyc delete mode 100644 test/bytecode_3.5/02_slice.pyc create mode 100644 test/bytecode_3.5_run/02_slice.pyc create mode 100644 test/bytecode_3.6_run/02_slice.pyc diff --git a/test/bytecode_2.7_run/02_slice.pyc b/test/bytecode_2.7_run/02_slice.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f3fd09398aa1a3fd3d120cf75c0ca15c6b75b8f2 GIT binary patch literal 323 zcmYk0F>69G5Xb*Xd?F9+A_c`k$2YabrAsMx=;|Ukh#+EuJbd;^E)K4R;^24bm+Kek zT?lGOe#u>axt#gy|946!X+C*u2C$Yzc6@Co*$>x#{dk@KOfzdpUMeZd*rKtm3x+#f2pqWb7J?wP8R`7&O`!_(muTI{^-YkRERyZlFO O#-qtqPv>f2y6PXZPc@1F literal 0 HcmV?d00001 diff --git a/test/bytecode_3.2/02_slice.pyc b/test/bytecode_3.2/02_slice.pyc deleted file mode 100644 index da9ccee1e7a057ad8563040ca676227c45eacafc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 227 zcmYj~F$=;l5QQ&EBT}@RgI&62(&8-Q$}?>+9q zW}AP?vgzpWXw+A{cmpi1AS=l#vKhTJHAD`gX8kQ`FStWQFC7G=VJp}$9%hA6UN9s> zF){`(yBl^Ai_d8`_-H|@&_>^n!L@NnQ|FE`c(6XTzv3{?X^qUeA8ZQl!m}dp+sQ%{ F@(UZmB4_{r diff --git a/test/bytecode_3.2_run/02_slice.pyc b/test/bytecode_3.2_run/02_slice.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7c23102ab5e366a38d66cc8f5265b87443996b6 GIT binary patch literal 339 zcmYk0v1-FG5Qa~Z9YS%7p#&N-WcAdR&TUDDtXT>jLcthIxTbO)q-gM3$P;A9Tl5+8 zpBoYj>Em>K`cF2wzWPZ1K1&*`81(|j9{`OlNK4X+bVOB5Ya)YY%>A9D=~|G1(R-Do zZ%O`zwvv9)|9>10y>pW4)vtjH8$*z)&Z!7ibV}WyRE&PlM)+GT7?PnF5rcof%J?1_ zh3f)qMYsXqyfttq#__ R@;sja6P1cPnTnaZ6-SP$HyZ!| literal 0 HcmV?d00001 diff --git a/test/bytecode_3.3/02_slice.pyc b/test/bytecode_3.3/02_slice.pyc deleted file mode 100644 index 0d1a3d43e76bf2c3a696b0268ffda6c3acb5426c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 198 zcmbQo!^_neRT(D7z`&3U#Bjh2WH|tFu|AMU0U}0*6efmrW`-yRhEyPz86?um#30EC z;sGU@Av_=*tib}*lgS7om_P)`EI$npzZAqwEUMH1QVa;9R1wJ4(=X10@;&dCIl21fBTZlX-=vg$h2aRJ^YM(i~txh9~uAv diff --git a/test/bytecode_3.3_run/02_slice.pyc b/test/bytecode_3.3_run/02_slice.pyc new file mode 100644 index 0000000000000000000000000000000000000000..94d9b3e3ca45735175cb68c930c05e4c9901e96b GIT binary patch literal 343 zcmYk0!D<3A6h&_mr<5`kp%g4#bUQ06UE9(w+`185NMR5Ybm-_vVis--{emvK>nHTH z{0{G%%4kRq$>Zd{_nTf{P13*jJ%Bk+vl8B3dBP82xfQI2HLxu#5=;TdVj}e~jk~FW zqfvVcjSr3RjIG8e>;IpVbFWDI3Ygp8H&}Wba|+vpS#CU7G(b^uP?pLE?&|Z XtSTEz_agm5O5B=`>gtZVH#h142Cz6Y literal 0 HcmV?d00001 diff --git a/test/bytecode_3.4/02_slice.pyc b/test/bytecode_3.4/02_slice.pyc deleted file mode 100644 index fbf24a3778346ab205666a3fba167d7146f165ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 232 zcmX|5I}XAy40Rd?g!s%X>jJT0KkLrH%{4-V38;$XAnn_aM%Gzk~~zd5%gJ5)Z{-!aXoC*jHc=+ zP*}7YvK>Dde*@DX`K9*;tq~|ICry4(ql>VZr0!H6(P(k@Sa^5R`KZi=&I+&O44t$N mZJHHI?yWA3kZ4s(Jelsl$++#`4HY&dcN diff --git a/test/bytecode_3.4_run/02_slice.pyc b/test/bytecode_3.4_run/02_slice.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f06f157af6ddc1dd805924207bdae1ff365cf536 GIT binary patch literal 319 zcmaFI!^;)Mvo7Wt0|UcjAcg~0Aj<)Wi>CvL6d+<`NMT|~VP;5SVMu3Xh+<$!1q!l) z#9NsdB%>G^Qh_SK>e87QvY8l)Z5c|K7@C0M5Y4Q}>fk0ZgOz3=DFvzqS^_kc8Ez`j zU}grWsgt2ffmR1=vb|&ky6zSyv1RYo1apelWGSt Vx|jz@h%yT>N-zm9N;3;H0sxq)K0^Qi literal 0 HcmV?d00001 diff --git a/test/bytecode_3.5/02_slice.pyc b/test/bytecode_3.5/02_slice.pyc deleted file mode 100644 index f884900759435563d8e922235e8088e54d75423d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 213 zcmWgR<>mS#B^4FUz`*brh~a=0$Z`PUVmBa>0z`}qDNGD0%nT_k4C$;4Q49>JKtWcJ zcqrYY;+K4JW^O@FYJ72iX;E^jesNAFkTfug2Q%~vDsOSvd3GL@_X=0tHz? z;;l>!l2MEdsX!HAb?HnD*-Q+@whSdq3{60Bh-Ow~b#RlI!Adiblmb-)EdiR!3^x^M zFf#+x)X7k#K&yi_*OlnfZCH zMMe2VD;bJ7fGWVmFZtrk+=86c_~QK1qU2Ql;+#w%Xdhpt1}=Qy5#Bp?Vopn8A9}K(Z;U!3>&g zFByRrzGMOs%pigVM1Wl5r^y_}oLE$Oi!t#QpJQ=xYEemMex7SlQGU@%h9VB23NZ0Y zzBn_tASX4xIKQ+gIaR+nClg2-7{!AbdIgoYIBatBQ%ZAE?LbBs^8g7xW*$ZXCLTs1 HW