diff --git a/test/bytecode_3.3/10_while1_popblock.pyc b/test/bytecode_3.3/10_while1_popblock.pyc new file mode 100644 index 00000000..1dd6eb6d Binary files /dev/null and b/test/bytecode_3.3/10_while1_popblock.pyc differ diff --git a/test/bytecode_3.4/10_while1_popblock.pyc b/test/bytecode_3.4/10_while1_popblock.pyc new file mode 100644 index 00000000..e8eb908d Binary files /dev/null and b/test/bytecode_3.4/10_while1_popblock.pyc differ diff --git a/test/bytecode_3.6/09_ext_arg_jump.pyc b/test/bytecode_3.6/09_ext_arg_jump.pyc new file mode 100644 index 00000000..65e5e805 Binary files /dev/null and b/test/bytecode_3.6/09_ext_arg_jump.pyc differ diff --git a/test/simple_source/bug33/10_while1_popblock.py b/test/simple_source/bug33/10_while1_popblock.py new file mode 100644 index 00000000..3219141b --- /dev/null +++ b/test/simple_source/bug33/10_while1_popblock.py @@ -0,0 +1,17 @@ +# From 3.4.4 mailcap.py +# Bug was needing a grammar rule to add POP_BLOCK before the end of the while1. +# 3.3 apparently doesn't add this. +def readmailcapfile(line): + while 1: + if not line: break + if line[0] == '#' or line.strip() == '': + continue + if not line: + continue + for j in range(3): + line[j] = line[j].strip() + if '/' in line: + line['/'].append('a') + else: + line['/'] = 'a' + return diff --git a/test/simple_source/bug36/03_fn_defaults.py b/test/simple_source/bug36/03_fn_defaults.py index 172d61d1..6167ce0c 100644 --- a/test/simple_source/bug36/03_fn_defaults.py +++ b/test/simple_source/bug36/03_fn_defaults.py @@ -7,3 +7,9 @@ def foo3(bar, baz=1, qux=2): return 3 def foo4(bar, baz, qux=1, quux=2): return 4 + +# From 3.6 compileall. +# Bug was in omitting default which when used in an "if" +# are treated as False would be +def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): + return diff --git a/test/simple_source/bug36/09_ext_arg_jump.py b/test/simple_source/bug36/09_ext_arg_jump.py new file mode 100644 index 00000000..d14cc98a --- /dev/null +++ b/test/simple_source/bug36/09_ext_arg_jump.py @@ -0,0 +1,37 @@ +# From 3.6 argparse. Bug was in handling EXTENDED_ARGS in a JUMP_FORWARD +# messing up control flow detection +def _format_usage(self, usage, actions, groups, prefix): + if usage: + usage = usage % dict(prog=self._prog) + + elif usage is None: + prog = 5 + + for action in actions: + if action.option_strings: + actions.append(action) + else: + actions.append(action) + + action_usage = format(optionals + positionals, groups) + + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + else: + if len(lines) > 1: + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + usage = '\n'.positionals(lines) + + return diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index e1259a9a..1b8cdc8d 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -1,4 +1,17 @@ -# Copyright (c) 2017 Rocky Bernstein +# Copyright (c) 2017-2018 Rocky Bernstein + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . """ spark grammar differences over Python 3.3 for Python 3.4 """ @@ -18,6 +31,10 @@ class Python34Parser(Python33Parser): expr ::= LOAD_ASSERT + # Seems to be needed starting 3.4.4 or so + while1stmt ::= SETUP_LOOP l_stmts + COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP + # FIXME the below masks a bug in not detecting COME_FROM_LOOP # grammar rules with COME_FROM -> COME_FROM_LOOP already exist whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK @@ -30,8 +47,10 @@ class Python34Parser(Python33Parser): """ def customize_grammar_rules(self, tokens, customize): - # self.remove_rules(""" - # """) + self.remove_rules(""" + # 3.4.2 has this. 3.4.4 may now + # while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + """) super(Python34Parser, self).customize_grammar_rules(tokens, customize) return diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 0e7d7074..3229acaa 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -100,12 +100,15 @@ class Scanner(object): That is, it is ether "JUMP_FORWARD" or an absolute jump that goes forward. """ - if self.code[offset] == self.opc.JUMP_FORWARD: + opname = self.get_inst(offset).opname + if opname == 'JUMP_FORWARD': return True - if self.code[offset] != self.opc.JUMP_ABSOLUTE: + if opname != 'JUMP_ABSOLUTE': return False - # FIXME 0 isn't always correct - return offset < self.get_target(offset, 0) + return offset < self.get_target(offset) + + def prev_offset(self, offset): + return self.insts[self.offset2inst_index[offset]-1].offset def get_inst(self, offset): # Instructions can get moved as a result of EXTENDED_ARGS removal. diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index b410653e..469a1fec 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -295,9 +295,22 @@ class Scanner2(Scanner): target = self.get_target(offset) if target <= offset: op_name = 'JUMP_BACK' - if (offset in self.stmts - and self.code[offset+3] not in (self.opc.END_FINALLY, - self.opc.POP_BLOCK)): + + # 'Continue's include jumps to loops that are not + # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP. + # If the JUMP_ABSOLUTE is + # either to a FOR_ITER or the instruction after a SETUP_LOOP + # and it is followed by another JUMP_FORWARD + # then we'll take it as a "continue". + j = self.offset2inst_index[offset] + target_index = self.offset2inst_index[target] + is_continue = (self.insts[target_index-1].opname == 'SETUP_LOOP' + and self.insts[j+1].opname == 'JUMP_FORWARD') and False + if is_continue: + op_name = 'CONTINUE' + if (offset in self.stmts and + self.code[offset+3] not in (self.opc.END_FINALLY, + self.opc.POP_BLOCK)): if ((offset in self.linestartoffsets and self.code[self.prev[offset]] == self.opc.JUMP_ABSOLUTE) or self.code[target] == self.opc.FOR_ITER diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 04cee6f8..edcfcb4b 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -425,11 +425,21 @@ class Scanner3(Scanner): target = self.get_target(inst.offset) if target <= inst.offset: next_opname = self.insts[i+1].opname - if (inst.offset in self.stmts and + + # 'Continue's include jumps to loops that are not + # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP. + # If the JUMP_ABSOLUTE is to a FOR_ITER and it is followed by another JUMP_FORWARD + # then we'll take it as a "continue". + is_continue = (self.insts[self.offset2inst_index[target]] + .opname == 'FOR_ITER' + and self.insts[i+1].opname == 'JUMP_FORWARD') + + if (is_continue or + (inst.offset in self.stmts and (self.version != 3.0 or (hasattr(inst, 'linestart'))) and (next_opname not in ('END_FINALLY', 'POP_BLOCK', # Python 3.0 only uses POP_TOP - 'POP_TOP'))): + 'POP_TOP')))): opname = 'CONTINUE' else: opname = 'JUMP_BACK' diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 60be5582..8da2c515 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -456,26 +456,28 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None): # MAKE_CLOSURE adds an additional closure slot - # Thank you, Python, for a such a well-thought out system that has - # changed 4 or so times. + # Thank you, Python: such a well-thought out system that has + # changed and continues to change many times. def build_param(ast, name, default): """build parameters: - handle defaults - handle format tuple parameters """ - if default: - if self.version >= 3.6: - value = default - else: - value = self.traverse(default, indent='') - maybe_show_tree_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result + if self.version >= 3.6: + value = default else: - return name + value = self.traverse(default, indent='') + maybe_show_tree_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + + # The below can probably be removed. This is probably + # a holdover from days when LOAD_CONST erroneously + # didn't handle LOAD_CONST None properly + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + + return result # MAKE_FUNCTION_... or MAKE_CLOSURE_... assert node[-1].kind.startswith('MAKE_') @@ -550,9 +552,14 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None): indent = self.indent # build parameters - tup = [paramnames, defparams] - params = [build_param(ast, name, default_value) for - name, default_value in map(lambda *tup:tup, *tup)] + params = [] + if defparams: + for i, defparam in enumerate(defparams): + params.append(build_param(ast, paramnames[i], defparam)) + + params += paramnames[i+1:] + else: + params = paramnames if not 3.0 <= self.version <= 3.1 or self.version >= 3.6: params.reverse() # back to correct order