From 8d1c454376e1aab4a174d98c2b178bac8b874c75 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 1 Mar 2018 17:23:58 -0500 Subject: [PATCH 1/7] small test tweak.. Allow it to run not under pytest --- pytest/test_token.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest/test_token.py b/pytest/test_token.py index fb0a2829..7c5cb8c0 100644 --- a/pytest/test_token.py +++ b/pytest/test_token.py @@ -18,3 +18,6 @@ def test_token(): t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True) expect = ' 1 LOAD_CONST 0 False' assert t.format() == expect + +if __name__ == '__main__': + test_token() From 452d17a6c3accfaf5e0277de68786dfc298b3851 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 1 Mar 2018 18:56:08 -0500 Subject: [PATCH 2/7] 3.4 while1 bug fix --- test/bytecode_3.3/10_while1_popblock.pyc | Bin 0 -> 577 bytes test/bytecode_3.4/10_while1_popblock.pyc | Bin 0 -> 453 bytes .../simple_source/bug33/10_while1_popblock.py | 17 +++++++++++++ uncompyle6/parsers/parse34.py | 23 +++++++++++++++--- 4 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 test/bytecode_3.3/10_while1_popblock.pyc create mode 100644 test/bytecode_3.4/10_while1_popblock.pyc create mode 100644 test/simple_source/bug33/10_while1_popblock.py diff --git a/test/bytecode_3.3/10_while1_popblock.pyc b/test/bytecode_3.3/10_while1_popblock.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1dd6eb6d74914ce3eebd64c0f4c8f526afb2f9a2 GIT binary patch literal 577 zcmbVHze~eF82v73qezQ_4%$(dxL8YZE28LJ1P7rYB~8wHZJJzjJ*-ftIyyOsi~foJ z4ZiOp=+Mc#+~a*e-k0xv*6JJK>wE7@5XJQ8c;IJvW($zu6ZA~f1M%oO^pT7qafe&0 zco%r)12BK11!$eFGGYep9%N&WjL-3iXo;-Pnc{?|K9zmu{D;p&nh8oOv=nrC+n^`G zvIWMJ(lY2wyL6LOxik)_cyYg~qQ|2oV5z`lFx1i2;efFF8aC`I09kf$V3{g*(%ghR;9V?=2C~bgTq_xbFke%BKAXS-kx>1OT9N0rJqb53)eUsZFmd$}3w%g6t zasT07rLxu6xgMl>JU-Hob`#x&jFL&D(m2w0XyD1F0U+UvfGT2N1nzE&T~QM|UR|t` bbwl>Gs`9(se>Pml)XpSNY${LPYP{JuAe?M_ literal 0 HcmV?d00001 diff --git a/test/bytecode_3.4/10_while1_popblock.pyc b/test/bytecode_3.4/10_while1_popblock.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8eb908d544dac1d6ae45f0310ba2d6909dbc623 GIT binary patch literal 453 zcmYjNF;Buk6n?MQR$~B7bWlcJ=pqv0MvO*hV!}W|Vo2$oSSamrClM2zaCCCf$=$zn zs}l=9z{z*51J`@6-+SNpy}Nc@D(}?ZKYMEg_<@b(5Wb|)dz3hBgjawb0S7t(i%6d@ zt8*!?DfAbm`3EcrmfduuffdV9 z-C!6n2CN(qpm4=x^~pFjI}h$TF5DyS;Nwu9oG VAGndq#F_2ZB;&J0T;$X2us=cQS^EG0 literal 0 HcmV?d00001 diff --git a/test/simple_source/bug33/10_while1_popblock.py b/test/simple_source/bug33/10_while1_popblock.py new file mode 100644 index 00000000..3219141b --- /dev/null +++ b/test/simple_source/bug33/10_while1_popblock.py @@ -0,0 +1,17 @@ +# From 3.4.4 mailcap.py +# Bug was needing a grammar rule to add POP_BLOCK before the end of the while1. +# 3.3 apparently doesn't add this. +def readmailcapfile(line): + while 1: + if not line: break + if line[0] == '#' or line.strip() == '': + continue + if not line: + continue + for j in range(3): + line[j] = line[j].strip() + if '/' in line: + line['/'].append('a') + else: + line['/'] = 'a' + return diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index e1259a9a..6ec70252 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -1,4 +1,17 @@ -# Copyright (c) 2017 Rocky Bernstein +# Copyright (c) 2017-2018 Rocky Bernstein + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . """ spark grammar differences over Python 3.3 for Python 3.4 """ @@ -18,6 +31,9 @@ class Python34Parser(Python33Parser): expr ::= LOAD_ASSERT + while1stmt ::= SETUP_LOOP l_stmts + COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP + # FIXME the below masks a bug in not detecting COME_FROM_LOOP # grammar rules with COME_FROM -> COME_FROM_LOOP already exist whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK @@ -30,8 +46,9 @@ class Python34Parser(Python33Parser): """ def customize_grammar_rules(self, tokens, customize): - # self.remove_rules(""" - # """) + self.remove_rules(""" + while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + """) super(Python34Parser, self).customize_grammar_rules(tokens, customize) return From 99d9beac768783ab7d243f9073b84bbaad2cc738 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 1 Mar 2018 21:23:44 -0500 Subject: [PATCH 3/7] Code generation changes between 3.4.2 and 3.4.4 --- uncompyle6/parsers/parse34.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index 6ec70252..1b8cdc8d 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -31,6 +31,7 @@ class Python34Parser(Python33Parser): expr ::= LOAD_ASSERT + # Seems to be needed starting 3.4.4 or so while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP @@ -47,7 +48,8 @@ class Python34Parser(Python33Parser): def customize_grammar_rules(self, tokens, customize): self.remove_rules(""" - while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + # 3.4.2 has this. 3.4.4 may now + # while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP """) super(Python34Parser, self).customize_grammar_rules(tokens, customize) return From 680701552677bbec4f1ffe7d7c3af95b4732927b Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 1 Mar 2018 22:47:36 -0500 Subject: [PATCH 4/7] Better CONTINUE detection on 3.x Helps when line numbers have been stripped say in optimization --- uncompyle6/scanners/scanner3.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 3253139b..c2580e61 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -419,11 +419,21 @@ class Scanner3(Scanner): target = self.get_target(inst.offset) if target <= inst.offset: next_opname = self.insts[i+1].opname - if (inst.offset in self.stmts and + + # Continues include jumps to FOR_ITER that are not + # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP. + # If the JUMP_ABSOLUTE is ot a FOR_ITER and is followed by another JUMP_FORWARD + # then we'll take it as a "continue". + is_continue = (self.insts[self.offset2inst_index[target]] + .opname == 'FOR_ITER' + and self.insts[i+1].opname == 'JUMP_FORWARD') + + if (is_continue or + (inst.offset in self.stmts and (self.version != 3.0 or (hasattr(inst, 'linestart'))) and (next_opname not in ('END_FINALLY', 'POP_BLOCK', # Python 3.0 only uses POP_TOP - 'POP_TOP'))): + 'POP_TOP')))): opname = 'CONTINUE' else: opname = 'JUMP_BACK' From 26e1df835c639f975f48326b1b08abe6691b7130 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 1 Mar 2018 23:34:26 -0500 Subject: [PATCH 5/7] Better "continue" detection for 2.7 --- uncompyle6/scanners/scanner2.py | 19 ++++++++++++++++--- uncompyle6/scanners/scanner3.py | 4 ++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index a5627077..2fb6f207 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -291,9 +291,22 @@ class Scanner2(Scanner): target = self.get_target(offset) if target <= offset: op_name = 'JUMP_BACK' - if (offset in self.stmts - and self.code[offset+3] not in (self.opc.END_FINALLY, - self.opc.POP_BLOCK)): + + # 'Continue's include jumps to loops that are not + # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP. + # If the JUMP_ABSOLUTE is + # either to a FOR_ITER or the instruction after a SETUP_LOOP + # and it is followed by another JUMP_FORWARD + # then we'll take it as a "continue". + j = self.offset2inst_index[offset] + target_index = self.offset2inst_index[target] + is_continue = (self.insts[target_index-1].opname == 'SETUP_LOOP' + and self.insts[j+1].opname == 'JUMP_FORWARD') and False + if is_continue: + op_name = 'CONTINUE' + if (offset in self.stmts and + self.code[offset+3] not in (self.opc.END_FINALLY, + self.opc.POP_BLOCK)): if ((offset in self.linestartoffsets and self.code[self.prev[offset]] == self.opc.JUMP_ABSOLUTE) or self.code[target] == self.opc.FOR_ITER diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index c2580e61..922c739b 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -420,9 +420,9 @@ class Scanner3(Scanner): if target <= inst.offset: next_opname = self.insts[i+1].opname - # Continues include jumps to FOR_ITER that are not + # 'Continue's include jumps to loops that are not # and the end of a block which follow with POP_BLOCK and COME_FROM_LOOP. - # If the JUMP_ABSOLUTE is ot a FOR_ITER and is followed by another JUMP_FORWARD + # If the JUMP_ABSOLUTE is to a FOR_ITER and it is followed by another JUMP_FORWARD # then we'll take it as a "continue". is_continue = (self.insts[self.offset2inst_index[target]] .opname == 'FOR_ITER' From 8d503682b3647743de4e46c7cb2ef66c36875cb6 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 2 Mar 2018 07:15:23 -0500 Subject: [PATCH 6/7] Use get_inst and self.insts more.. needed more in 3.6 to handle EXTENDED_ARGS before JUMP_xxx --- test/bytecode_3.6/09_ext_arg_jump.pyc | Bin 0 -> 803 bytes test/simple_source/bug36/09_ext_arg_jump.py | 37 ++++++++++++++++++++ uncompyle6/scanner.py | 11 +++--- 3 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 test/bytecode_3.6/09_ext_arg_jump.pyc create mode 100644 test/simple_source/bug36/09_ext_arg_jump.py diff --git a/test/bytecode_3.6/09_ext_arg_jump.pyc b/test/bytecode_3.6/09_ext_arg_jump.pyc new file mode 100644 index 0000000000000000000000000000000000000000..65e5e805a07269defe002e3afc49e0f72d8f6546 GIT binary patch literal 803 zcmZ8eO^eh(5UuJYJ(FisCK;Xp}RJ3N)-x;AcF93NJCGzJRrRNvL3zvpd%8CA>YV zxaKP+pKG?VG_Mii;=%s=8&I}#6rX~&YF70~)r32P%2j=B*GL&_B-~xwM*V-+FN#mm7VV*~+X!tUd+fe!a4i7hlKghEpF~jDA2Okol-ut(LMJZ*J zsvt=PmB!jcY0C=oP%09PcFG9$<46UYB#jEcPeL6!A2fCmZI-@M7GWIuy>MX+h!xAM z%&F!wS|&T$?w)t3KkRDIaXdv1-EJEGZxc>y^cCa*o%;HIk! text_width: + + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + else: + if len(lines) > 1: + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + usage = '\n'.positionals(lines) + + return diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index f8ac665e..09d5999f 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -102,12 +102,15 @@ class Scanner(object): That is, it is ether "JUMP_FORWARD" or an absolute jump that goes forward. """ - if self.code[offset] == self.opc.JUMP_FORWARD: + opname = self.get_inst(offset).opname + if opname == 'JUMP_FORWARD': return True - if self.code[offset] != self.opc.JUMP_ABSOLUTE: + if opname != 'JUMP_ABSOLUTE': return False - # FIXME 0 isn't always correct - return offset < self.get_target(offset, 0) + return offset < self.get_target(offset) + + def prev_offset(self, offset): + return self.insts[self.offset2inst_index[offset]-1].offset def get_inst(self, offset): # Instructions can get moved as a result of EXTENDED_ARGS removal. From bb13988126355eba3dfa90dda011dc7db87a7198 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 2 Mar 2018 08:03:51 -0500 Subject: [PATCH 7/7] Instruction fixup broken 3.x make_func... for handling default values --- test/simple_source/bug36/03_fn_defaults.py | 6 ++++ uncompyle6/semantics/make_function.py | 38 +++++++++++++--------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/test/simple_source/bug36/03_fn_defaults.py b/test/simple_source/bug36/03_fn_defaults.py index 172d61d1..6167ce0c 100644 --- a/test/simple_source/bug36/03_fn_defaults.py +++ b/test/simple_source/bug36/03_fn_defaults.py @@ -7,3 +7,9 @@ def foo3(bar, baz=1, qux=2): return 3 def foo4(bar, baz, qux=1, quux=2): return 4 + +# From 3.6 compileall. +# Bug was in omitting default which when used in an "if" +# are treated as False would be +def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): + return diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 1dc8e119..e0750ff1 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -459,26 +459,28 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None): # MAKE_CLOSURE adds an additional closure slot - # Thank you, Python, for a such a well-thought out system that has - # changed 4 or so times. + # Thank you, Python: such a well-thought out system that has + # changed and continues to change many times. def build_param(ast, name, default): """build parameters: - handle defaults - handle format tuple parameters """ - if default: - if self.version >= 3.6: - value = default - else: - value = self.traverse(default, indent='') - maybe_show_tree_param_default(self.showast, name, value) - result = '%s=%s' % (name, value) - if result[-2:] == '= ': # default was 'LOAD_CONST None' - result += 'None' - return result + if self.version >= 3.6: + value = default else: - return name + value = self.traverse(default, indent='') + maybe_show_tree_param_default(self.showast, name, value) + result = '%s=%s' % (name, value) + + # The below can probably be removed. This is probably + # a holdover from days when LOAD_CONST erroneously + # didn't handle LOAD_CONST None properly + if result[-2:] == '= ': # default was 'LOAD_CONST None' + result += 'None' + + return result # MAKE_FUNCTION_... or MAKE_CLOSURE_... assert node[-1].kind.startswith('MAKE_') @@ -552,8 +554,14 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None): kw_pairs = args_node.attr[1] if self.version >= 3.0 else 0 # build parameters - params = [build_param(ast, name, d) for - name, d in zip_longest(paramnames, defparams, fillvalue=None)] + params = [] + if defparams: + for i, defparam in enumerate(defparams): + params.append(build_param(ast, paramnames[i], defparam)) + + params += paramnames[i+1:] + else: + params = paramnames if not 3.0 <= self.version <= 3.1 or self.version >= 3.6: params.reverse() # back to correct order