From 1e324e0e8db0d97c1ea63c577805a7b00720b83d Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 26 Nov 2016 21:41:45 -0500 Subject: [PATCH] Misc changes scanner26.py: make scanner2.py and scanner26.py more alike scanner2.py: check that return stmt is last in list. (May change) main.py: show filename on verify error test/*: add more --- test/bytecode_2.6/01_boolean.pyc | Bin 157 -> 0 bytes test/bytecode_2.6/03_elif_vs_continue.pyc | Bin 0 -> 534 bytes .../bug26/03_elif_vs_continue.py | 18 ++++++ uncompyle6/main.py | 7 +- uncompyle6/parser.py | 4 +- uncompyle6/parsers/parse2.py | 11 +++- uncompyle6/scanners/scanner2.py | 61 +++++++++--------- uncompyle6/scanners/scanner26.py | 5 +- 8 files changed, 70 insertions(+), 36 deletions(-) delete mode 100644 test/bytecode_2.6/01_boolean.pyc create mode 100644 test/bytecode_2.6/03_elif_vs_continue.pyc create mode 100644 test/simple_source/bug26/03_elif_vs_continue.py diff --git a/test/bytecode_2.6/01_boolean.pyc b/test/bytecode_2.6/01_boolean.pyc deleted file mode 100644 index d65f034709f11eccfeb7e7fd37fcb0f541dc47dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 157 zcmcckiI?lqw~DZ21}I41TPoKQ%ZAE?LbBsGXV(}MgW{wA6EbX diff --git a/test/bytecode_2.6/03_elif_vs_continue.pyc b/test/bytecode_2.6/03_elif_vs_continue.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74189e25bbcd4c5e18de63398fce25a8dcfe1bb6 GIT binary patch literal 534 zcmbVJO-sW-5Pg%TABGk@6#s!3JXlfiAW{VDO?qe#iYOr_+q%+ZV|G(2Bp3A$`d9n~ z&aMUTE<0~#-p=g2O}@Sd!Fd1tF2Qzr>@V5$E<;L4fltCc2ZVWOPZOV^%Vzx*n{F6t z4-#m6)Ggcd(t`>>+|;y?6KHdT8kjsz@Zh0oqv@b-qb67BgI*6nmhc=gl8xiEbWW>s!vCa%j;m7#rz zv5-8+r5486{>3U@YS)ygRQytMQx%R9nKUc}P=#~xI^+Zn^dU!DX2nWGT2*BtMo-o9 z#x>?U)LA1Tykod5s; literal 0 HcmV?d00001 diff --git a/test/simple_source/bug26/03_elif_vs_continue.py b/test/simple_source/bug26/03_elif_vs_continue.py new file mode 100644 index 00000000..bcc27bfd --- /dev/null +++ b/test/simple_source/bug26/03_elif_vs_continue.py @@ -0,0 +1,18 @@ +# Bug was using continue fouling up 1st elif, by confusing +# the "pass" for "continue" by not recognizing the if jump +# around it. We fixed by ignoring what's done in Python 2.7 +# Better is better detection of control structures + +def _compile_charset(charset, flags, code, fixup=None): + # compile charset subprogram + emit = code.append + if fixup is None: + fixup = 1 + for op, av in charset: + if op is flags: + pass + elif op is code: + emit(fixup(av)) + else: + raise RuntimeError + emit(5) diff --git a/uncompyle6/main.py b/uncompyle6/main.py index 9c3de62a..256958a0 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -189,17 +189,16 @@ def main(in_base, out_base, files, codes, outfile=None, print(e) verify_failed_files += 1 os.rename(outfile, outfile + '_unverified') + sys.stderr.write("### Error Verifying %s\n" % filename) + sys.stderr.write(str(e) + "\n") if not outfile: - print("### Error Verifiying %s" % filename, file=sys.stderr) - print(e, file=sys.stderr) if raise_on_error: raise pass pass pass elif do_verify: - print("\n### uncompile successful, but no file to compare against", - file=sys.stderr) + sys.stderr.write("\n### uncompile successful, but no file to compare against\n") pass else: okay_files += 1 diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index ee9abcb1..5909e95e 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -81,7 +81,7 @@ class PythonParser(GenericASTBuilder): else: prefix = ' ' if hasattr(p_token, 'offset'): - prefix += "%3d " % p_token.offset + prefix += "%3s " % str(p_token.offset) prefix += " " else: prefix = ' ' @@ -485,6 +485,8 @@ class PythonParser(GenericASTBuilder): _mklambda ::= load_closure mklambda _mklambda ::= mklambda + # "and" where the first part of the and is true, + # so there is only the 2nd part to evaluate and2 ::= _jump jmp_false COME_FROM expr COME_FROM expr ::= conditional diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 6778f014..00565e4c 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -392,13 +392,22 @@ class Python2Parser(PythonParser): pass self.check_reduce['augassign1'] = 'AST' self.check_reduce['augassign2'] = 'AST' + self.check_reduce['_stmts'] = 'AST' return def reduce_is_invalid(self, rule, ast, tokens, first, last): lhs = rule[0] if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and': return True - # Add more stuff, like COME_FROM checking + elif lhs == '_stmts': + for i, stmt in enumerate(ast): + if stmt == '_stmts': + stmt = stmt[0] + assert stmt == 'stmt' + if stmt[0] == 'return_stmt': + return i+1 != len(ast) + pass + return False return False class Python2ParserSingle(Python2Parser, PythonParserSingle): diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 389f37ad..1a6e9ec4 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -166,9 +166,9 @@ class Scanner2(scan.Scanner): # continue # last_offset = jump_offset come_from_name = 'COME_FROM' - opname = self.opc.opname[self.code[jump_offset]] - if opname.startswith('SETUP_') and self.version == 2.7: - come_from_type = opname[len('SETUP_'):] + op_name = self.opc.opname[self.code[jump_offset]] + if op_name.startswith('SETUP_') and self.version == 2.7: + come_from_type = op_name[len('SETUP_'):] if come_from_type not in ('LOOP', 'EXCEPT'): come_from_name = 'COME_FROM_%s' % come_from_type pass @@ -179,7 +179,7 @@ class Scanner2(scan.Scanner): jump_idx += 1 op = self.code[offset] - opname = self.opc.opname[op] + op_name = self.opc.opname[op] oparg = None; pattr = None has_arg = op_has_argument(op, self.opc) @@ -194,14 +194,14 @@ class Scanner2(scan.Scanner): if iscode(const): oparg = const if const.co_name == '': - assert opname == 'LOAD_CONST' - opname = 'LOAD_LAMBDA' + assert op_name == 'LOAD_CONST' + op_name = 'LOAD_LAMBDA' elif const.co_name == '': - opname = 'LOAD_GENEXPR' + op_name = 'LOAD_GENEXPR' elif const.co_name == '': - opname = 'LOAD_DICTCOMP' + op_name = 'LOAD_DICTCOMP' elif const.co_name == '': - opname = 'LOAD_SETCOMP' + op_name = 'LOAD_SETCOMP' # verify() uses 'pattr' for comparison, since 'attr' # now holds Code(const) and thus can not be used # for comparison (todo: think about changing this) @@ -237,20 +237,20 @@ class Scanner2(scan.Scanner): self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE: continue else: - if self.is_pypy and not oparg and opname == 'BUILD_MAP': - opname = 'BUILD_MAP_n' + if self.is_pypy and not oparg and op_name == 'BUILD_MAP': + op_name = 'BUILD_MAP_n' else: - opname = '%s_%d' % (opname, oparg) + op_name = '%s_%d' % (op_name, oparg) if op != self.opc.BUILD_SLICE: - customize[opname] = oparg - elif self.is_pypy and opname in ('LOOKUP_METHOD', + customize[op_name] = oparg + elif self.is_pypy and op_name in ('LOOKUP_METHOD', 'JUMP_IF_NOT_DEBUG', 'SETUP_EXCEPT', 'SETUP_FINALLY'): # The value in the dict is in special cases in semantic actions, such # as CALL_FUNCTION. The value is not used in these cases, so we put # in arbitrary value 0. - customize[opname] = 0 + customize[op_name] = 0 elif op == self.opc.JUMP_ABSOLUTE: # Further classify JUMP_ABSOLUTE into backward jumps # which are used in loops, and "CONTINUE" jumps which @@ -269,16 +269,16 @@ class Scanner2(scan.Scanner): and self.code[offset+3] not in (self.opc.END_FINALLY, self.opc.POP_BLOCK) and offset not in self.not_continue): - opname = 'CONTINUE' + op_name = 'CONTINUE' else: - opname = 'JUMP_BACK' + op_name = 'JUMP_BACK' elif op == self.opc.LOAD_GLOBAL: if offset in self.load_asserts: - opname = 'LOAD_ASSERT' + op_name = 'LOAD_ASSERT' elif op == self.opc.RETURN_VALUE: if offset in self.return_end_ifs: - opname = 'RETURN_END_IF' + op_name = 'RETURN_END_IF' if offset in self.linestartoffsets: linestart = self.linestartoffsets[offset] @@ -287,7 +287,7 @@ class Scanner2(scan.Scanner): if offset not in replace: tokens.append(Token( - opname, oparg, pattr, offset, linestart, op, + op_name, oparg, pattr, offset, linestart, op, has_arg, self.opc)) else: tokens.append(Token( @@ -782,21 +782,23 @@ class Scanner2(scan.Scanner): if offset in self.ignore_if: return - if code[pre[rtarget]] == self.opc.JUMP_ABSOLUTE and pre[rtarget] in self.stmts \ - and pre[rtarget] != offset and pre[pre[rtarget]] != offset: - if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK: - if code[pre[pre[rtarget]]] != self.opc.JUMP_ABSOLUTE: - pass - elif self.get_target(pre[pre[rtarget]]) != target: - pass + if self.version == 2.7: + if code[pre[rtarget]] == self.opc.JUMP_ABSOLUTE and pre[rtarget] in self.stmts \ + and pre[rtarget] != offset and pre[pre[rtarget]] != offset: + if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK: + if code[pre[pre[rtarget]]] != self.opc.JUMP_ABSOLUTE: + pass + elif self.get_target(pre[pre[rtarget]]) != target: + pass + else: + rtarget = pre[rtarget] else: rtarget = pre[rtarget] - else: - rtarget = pre[rtarget] # Does the "if" jump just beyond a jump op, then this is probably an if statement pre_rtarget = pre[rtarget] code_pre_rtarget = code[pre_rtarget] + if code_pre_rtarget in self.jump_forward: if_end = self.get_target(pre_rtarget) @@ -824,6 +826,7 @@ class Scanner2(scan.Scanner): self.structs.append({'type': 'if-then', 'start': start-3, 'end': pre_rtarget}) + self.not_continue.add(pre_rtarget) if rtarget < end: diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index 4f65d9f1..fbd2764a 100755 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -233,7 +233,7 @@ class Scanner26(scan.Scanner2): if op != self.opc.BUILD_SLICE: customize[op_name] = oparg elif op == self.opc.JUMP_ABSOLUTE: - # Further classifhy JUMP_ABSOLUTE into backward jumps + # Further classify JUMP_ABSOLUTE into backward jumps # which are used in loops, and "CONTINUE" jumps which # may appear in a "continue" statement. The loop-type # and continue-type jumps will help us classify loop @@ -254,6 +254,9 @@ class Scanner26(scan.Scanner2): # if x: continue # the "continue" is not on a new line. if tokens[-1].type == 'JUMP_BACK': + # We need 'intern' since we have + # already have processed the previous + # token. tokens[-1].type = intern('CONTINUE') elif op == self.opc.LOAD_GLOBAL: