Get ready for release 2.9.9

* Python 3: break; continue -> break
* Use variable in detect_structure for pre[rtarget]
* Make Python 2 and Python 3 detect_structure more alie

ChangeLog

@@ -1,6 +1,223 @@
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/version.py: Get ready for release 2.9.9
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py: 2.x list_if may have a THEN in it
+
+2016-12-31  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Towards fixing a Python 3.3
+	return/continue bug
+
+2016-12-30  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/main.py: On --verify if we can't unbuffer output, don't
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	dectect_structure() -> detect_control_flow()
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	DRY code and emitted Python 3 source * Python 3: break; continue -> break * Use variable in detect_structure for pre[rtarget] * Make Python 2 and Python 3 detect_structure more alie
+
+2016-12-29  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: More if/then detection in Python
+	3.x
+
+2016-12-29  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #73 from rocky/then-crap Add THEN token to improve Python 2.2-2.6 control flow detection
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/scanners/tok.py: Misc
+	bugs
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* : commit 723fa5dfed5bb198c66741c594e2c277ded88970 Author: rocky
+	<rb@dustyfeet.com> Date:   Wed Dec 28 18:57:09 2016 -0500
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
+	uncompyle6/parsers/parse33.py: Towards fixing a 3.2 while true: ...
+	break bug
+
+2016-12-28  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/main.py, uncompyle6/parsers/parse26.py,
+	uncompyle6/verify.py: Bugs in Python 2.6- "and" and "lambda"
+	handling ..  and clean up verify output
+
+2016-12-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse26.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner26.py, uncompyle6/semantics/pysource.py: 
+	WIP : Add THEN to disambigute from "and"
+
+2016-12-27  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py: Make 2.6 and 2.7 ingest more
+	alike
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* : Update 2.7 bytecode file for last fix
+
+2016-12-26  R. Bernstein <rocky@users.noreply.github.com>
+
+	* : Merge pull request #71 from jiangpengcheng/tupple_bug tupples which contain only 1 element need a comma
+
+2016-12-26  jiangpch <jiangpch@gohighsec.com>
+
+	* uncompyle6/semantics/pysource.py: tupples which contain only 1
+	element need a comma
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse25.py: fix bug in using python2 AST rules
+	in python 2.5
+
+2016-12-26  rocky <rb@dustyfeet.com>
+
+	* : commit f1a947f106b231fb1480ba301b15e3ceaf78c94f Author: rocky
+	<rb@dustyfeet.com> Date:   Mon Dec 26 00:43:02 2016 -0500
+
+2016-12-25  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner23.py,
+	uncompyle6/scanners/scanner24.py, uncompyle6/semantics/pysource.py,
+	uncompyle6/verify.py: Scanner call fixes. NAME_MODULE removal for
+	<=2.4
+
+2016-12-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/astnode.py, uncompyle6/parsers/parse2.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner15.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner21.py,
+	uncompyle6/scanners/scanner22.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Lint
+
+2016-12-24  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Remove stray debug hook
+
+2016-12-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/pysource.py: Bang on 3.6
+	build_map_unpack_with_call Probably will fix better in the future.
+
+2016-12-18  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/bin/pydisassemble.py, uncompyle6/bin/uncompile.py,
+	uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse25.py,
+	uncompyle6/parsers/parse27.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner3.py: 
+	Python flake8 crap Was testing realgud's C-x!8 (goto flake8 warning/error)
+
+2016-12-18  rocky <rb@dustyfeet.com>
+
+	* pytest/.gitignore, test/simple_source/bug25/02_try_else.py,
+	uncompyle6/parsers/parse25.py, uncompyle6/parsers/parse26.py: Python
+	2.5 mistaken try/else
+
+2016-12-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner2.py, uncompyle6/scanners/scanner25.py: 
+	show-asm on python2.5 is optional make scanner2 look a little more like scanner3
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* NEWS: Release 2.9.8 news
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/version.py: Get ready for release 2.9.8
+
+2016-12-16  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/bug35/02_build_map_unpack_with_call.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/parsers/parse36.py, uncompyle6/scanners/scanner3.py,
+	uncompyle6/semantics/pysource.py: Start to handle 3.5
+	build_map_unpack_with_call 3.6 also started but needs even more work
+
+2016-12-15  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanner.py, uncompyle6/scanners/scanner3.py: Some
+	Python 3.6 bytecode->wordcode fixes
+
+2016-12-13  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py: Was passing wrong type
+
+2016-12-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parser.py: option -g: show start-end range when
+	possible
+
+2016-12-11  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/make_function.py, uncompyle6/verify.py: two
+	misc changes - track print_docstring move to help (used in python 3.1) - verify: allow RETURN_VALUE to match RETURN_END_IF
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* .travis.yml, test/Makefile: 3.2 needs --weak-verify
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* .travis.yml: Try testing on 3.2
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py, uncompyle6/bin/uncompile.py: Can run in Python 3.1
+	and Python 3.2
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* test/Makefile, uncompyle6/parsers/parse3.py,
+	uncompyle6/scanners/scanner3.py: Another python 3 ELSE fixes and ...  Makefile:   - test python 3.0 bytecode   - turn full --verify back on Python 3.x
+
+2016-12-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/scanner3.py: Another faulty Python3 ELSE tag
+	remove
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* pytest/test_grammar.py: Grammar check: ELSE on RHS is ok.
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/scanners/tok.py, uncompyle6/verify.py: Back of some of
+	the verification changes
+
+2016-12-09  rocky <rb@dustyfeet.com>
+
+	* : commit a5d2237435ee51e681c73db9e7ea379d56456205 Author: rocky
+	<rb@dustyfeet.com> Date:   Fri Dec 9 21:10:10 2016 -0500
+
 2016-12-04  rocky <rb@dustyfeet.com>
 
-	* uncompyle6/version.py: Get ready for release 2.9.7
+	* ChangeLog, NEWS, uncompyle6/main.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse3.py,
+	uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
+	uncompyle6/scanner.py, uncompyle6/scanners/scanner2.py,
+	uncompyle6/scanners/scanner23.py, uncompyle6/scanners/scanner24.py,
+	uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py,
+	uncompyle6/semantics/make_function.py,
+	uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
+	uncompyle6/version.py: Get ready for release 2.9.7 Some of the many lint things. Linting is kind of stupid though.
 
 2016-11-28  rocky <rb@dustyfeet.com>
 

NEWS

@@ -1,3 +1,18 @@
+uncompyle6 2.9.9 2016-12-16
+
+- Better control-flow detection
+- pseudo instruction THEN in 2.x
+  to disambiguate if from and
+- fix bug in --verify option
+- DRY (a little) control-flow detection
+- fix syntax in tuples with one element
+- if AST rule inheritence in Python 2.5
+- NAME_MODULE removal for Python <= 2.4
+- verifycall fixes for Python <= 2.4
+- more Python lint
+
+uncompyle6 2.9.8 2016-12-16
+
 uncompyle6 2.9.7 2016-12-16
 
 - Start to handle 3.5/3.6 build_map_unpack_with_call

test/Makefile

@@ -100,7 +100,7 @@ check-bytecode-2.5:
 
 #: Check deparsing Python 2.6
 check-bytecode-2.6:
-	$(PYTHON) test_pythonlib.py --bytecode-2.6
+	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:

uncompyle6/main.py

@@ -126,8 +126,9 @@ def main(in_base, out_base, files, codes, outfile=None,
                     prefix = prefix[:-len('.py')]
                 junk, outfile = tempfile.mkstemp(suffix=".py",
                                              prefix=prefix)
-                # Unbuffer output
-                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
+                # Unbuffer output if possible
+                buffering = -1 if sys.stdout.isatty() else 0
+                sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
                 tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
                 os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
                 os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
@@ -230,11 +231,11 @@ def status_msg(do_verify, tot_files, okay_files, failed_files,
                verify_failed_files):
     if tot_files == 1:
         if failed_files:
-            return "decompile failed"
+            return "\n# decompile failed"
         elif verify_failed_files:
-            return "decompile verify failed"
+            return "\n# decompile verify failed"
         else:
-            return "Successfully decompiled file"
+            return "\n# Successfully decompiled file"
             pass
         pass
     mess = "decompiled %i files: %i okay, %i failed" % (tot_files, okay_files, failed_files)

uncompyle6/parsers/parse26.py

@@ -113,16 +113,10 @@ class Python26Parser(Python2Parser):
 
         break_stmt ::= BREAK_LOOP JUMP_BACK
 
-        # Semantic actions want the else to be at position 3
-        ifelsestmt  ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
-        ifelsestmt  ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
-
         # Semantic actions want else_suitel to be at index 3
         ifelsestmtl ::= testexpr c_stmts_opt jb_cf_pop else_suitel
         ifelsestmtc ::= testexpr c_stmts_opt ja_cf_pop else_suitec
 
-        iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
-
         # Semantic actions want suite_stmts_opt to be at index 3
         withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                      POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
@@ -159,6 +153,29 @@ class Python26Parser(Python2Parser):
 
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
 
+        ifstmt         ::= testexpr_then _ifstmts_jump
+
+        # Semantic actions want the else to be at position 3
+        ifelsestmt     ::= testexpr      c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
+        ifelsestmt     ::= testexpr      c_stmts_opt filler else_suitel come_froms POP_TOP
+        ifelsestmt     ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
+
+        # Semantic actions want else_suitel to be at index 3
+        ifelsestmtl    ::= testexpr_then c_stmts_opt jb_cf_pop else_suitel
+        ifelsestmtc    ::= testexpr_then c_stmts_opt ja_cf_pop else_suitec
+
+        iflaststmt     ::= testexpr_then c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+        iflaststmt     ::= testexpr      c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
+
+        testexpr_then  ::= testtrue_then
+        testexpr_then  ::= testfalse_then
+        testtrue_then  ::= expr jmp_true_then
+        testfalse_then ::= expr jmp_false_then
+
+        jmp_false_then ::= JUMP_IF_FALSE THEN POP_TOP
+        jmp_true_then  ::= JUMP_IF_TRUE THEN POP_TOP
+
         # Common with 2.7
         while1stmt ::= SETUP_LOOP return_stmts bp_come_from
         while1stmt ::= SETUP_LOOP return_stmts COME_FROM
@@ -196,16 +213,16 @@ class Python26Parser(Python2Parser):
         genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop
                          jb_bp_come_from
         genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
-
+        list_if ::= list_if ::= expr jmp_false_then list_iter
         '''
 
     def p_ret26(self, args):
         '''
-        ret_and  ::= expr jmp_false ret_expr_or_cond COME_FROM
-        ret_or   ::= expr jmp_true ret_expr_or_cond COME_FROM
-        ret_cond ::= expr jmp_false expr RETURN_END_IF POP_TOP ret_expr_or_cond
-        ret_cond ::= expr jmp_false expr ret_expr_or_cond
-        ret_cond_not ::= expr jmp_true expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_and      ::= expr jmp_false ret_expr_or_cond COME_FROM
+        ret_or       ::= expr jmp_true ret_expr_or_cond COME_FROM
+        ret_cond     ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
+        ret_cond     ::= expr jmp_false_then expr ret_expr_or_cond
+        ret_cond_not ::= expr jmp_true_then  expr RETURN_END_IF POP_TOP ret_expr_or_cond
 
         return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
         return_stmt ::= ret_expr RETURN_VALUE POP_TOP
@@ -215,17 +232,37 @@ class Python26Parser(Python2Parser):
         '''
 
     def p_except26(self, args):
-        '''
+        """
         except_suite ::= c_stmts_opt jmp_abs POP_TOP
-        '''
+        """
 
     def p_misc26(self, args):
-        '''
+        """
         conditional  ::= expr jmp_false expr jf_cf_pop expr come_from_opt
         and  ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
         cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
-        '''
 
+        conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
+        """
+
+    def add_custom_rules(self, tokens, customize):
+        super(Python26Parser, self).add_custom_rules(tokens, customize)
+        self.check_reduce['and'] = 'AST'
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python26Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid:
+            return invalid
+        if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
+            # Test that jmp_false jumps to the end of "and"
+            # or that it jumps to the same place as the end of "and"
+            jmp_false = ast[1][0]
+            jmp_target = jmp_false.offset + jmp_false.attr + 3
+            return not (jmp_target == tokens[last].offset or
+                        tokens[last].pattr == jmp_false.pattr)
+        return False
 class Python26ParserSingle(Python2Parser, PythonParserSingle):
     pass
 

uncompyle6/scanners/scanner2.py

@@ -478,7 +478,7 @@ class Scanner2(scan.Scanner):
             elif op in self.setup_ops:
                 count_SETUP_ += 1
 
-    def detect_structure(self, offset, op):
+    def detect_control_flow(self, offset, op):
         """
         Detect type of block structures and their boundaries to fix optimized jumps
         in python2.3+
@@ -676,6 +676,8 @@ class Scanner2(scan.Scanner):
                 self.fixed_jumps[offset] = rtarget
                 return
 
+            jump_if_offset = offset
+
             start = offset+3
             pre = self.prev
 
@@ -698,6 +700,10 @@ class Scanner2(scan.Scanner):
                                        'end':   pre[target]})
                 return
 
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = pre[rtarget]
+
             # Is it an "and" inside an "if" or "while" block
             if op == self.opc.PJIF:
 
@@ -708,22 +714,22 @@ class Scanner2(scan.Scanner):
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    if code[pre[rtarget]] in self.jump_forward \
-                            and pre[rtarget] not in self.stmts \
-                            and self.restrict_to_parent(self.get_target(pre[rtarget]), parent) == rtarget:
-                        if code[pre[pre[rtarget]]] == self.opc.JUMP_ABSOLUTE \
+                    if code[pre_rtarget] in self.jump_forward \
+                            and pre_rtarget not in self.stmts \
+                            and self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget:
+                        if code[pre[pre_rtarget]] == self.opc.JUMP_ABSOLUTE \
                                 and self.remove_mid_line_ifs([offset]) \
-                                and target == self.get_target(pre[pre[rtarget]]) \
-                                and (pre[pre[rtarget]] not in self.stmts or self.get_target(pre[pre[rtarget]]) > pre[pre[rtarget]])\
-                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]], self.pop_jump_if, target))):
+                                and target == self.get_target(pre[pre_rtarget]) \
+                                and (pre[pre_rtarget] not in self.stmts or self.get_target(pre[pre_rtarget]) > pre[pre_rtarget])\
+                                and 1 == len(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget], self.pop_jump_if, target))):
                             pass
-                        elif code[pre[pre[rtarget]]] == self.opc.RETURN_VALUE \
+                        elif code[pre[pre_rtarget]] == self.opc.RETURN_VALUE \
                                 and self.remove_mid_line_ifs([offset]) \
                                 and 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start,
-                                                                                       pre[pre[rtarget]],
+                                                                                       pre[pre_rtarget],
                                                                                        self.pop_jump_if, target)))
-                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre[rtarget]],
-                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre[rtarget], True))))):
+                                              | set(self.remove_mid_line_ifs(self.rem_or(start, pre[pre_rtarget],
+                                                            (self.opc.PJIF, self.opc.PJIT, self.opc.JUMP_ABSOLUTE), pre_rtarget, True))))):
                             pass
                         else:
                             fix = None
@@ -756,7 +762,7 @@ class Scanner2(scan.Scanner):
                 else:
                     assert_offset = offset + 3
                 if (assert_offset) in self.load_asserts:
-                    if code[pre[rtarget]] == self.opc.RAISE_VARARGS:
+                    if code[pre_rtarget] == self.opc.RAISE_VARARGS:
                         return
                     self.load_asserts.remove(assert_offset)
 
@@ -765,7 +771,7 @@ class Scanner2(scan.Scanner):
                     pass
                 elif code[next] in self.jump_forward and target == self.get_target(next):
                     if code[pre[next]] == self.opc.PJIF:
-                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre[rtarget]]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
+                        if code[next] == self.opc.JUMP_FORWARD or target != rtarget or code[pre[pre_rtarget]] not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE):
                             self.fixed_jumps[offset] = pre[next]
                             return
                 elif code[next] == self.opc.JUMP_ABSOLUTE and code[target] in self.jump_forward:
@@ -782,17 +788,17 @@ class Scanner2(scan.Scanner):
                 return
 
             if self.version == 2.7:
-                if code[pre[rtarget]] == self.opc.JUMP_ABSOLUTE and pre[rtarget] in self.stmts \
-                        and pre[rtarget] != offset and pre[pre[rtarget]] != offset:
+                if code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and pre_rtarget in self.stmts \
+                        and pre_rtarget != offset and pre[pre_rtarget] != offset:
                     if code[rtarget] == self.opc.JUMP_ABSOLUTE and code[rtarget+3] == self.opc.POP_BLOCK:
-                        if code[pre[pre[rtarget]]] != self.opc.JUMP_ABSOLUTE:
+                        if code[pre[pre_rtarget]] != self.opc.JUMP_ABSOLUTE:
                             pass
-                        elif self.get_target(pre[pre[rtarget]]) != target:
+                        elif self.get_target(pre[pre_rtarget]) != target:
                             pass
                         else:
-                            rtarget = pre[rtarget]
+                            rtarget = pre_rtarget
                     else:
-                        rtarget = pre[rtarget]
+                        rtarget = pre_rtarget
 
             # Does the "jump if" jump beyond a jump op?
             # That is, we have something like:
@@ -808,7 +814,6 @@ class Scanner2(scan.Scanner):
             # There are other contexts we may need to consider
             # like whether the target is "END_FINALLY"
             # or if the condition jump is to a forward location
-            pre_rtarget = pre[rtarget]
             code_pre_rtarget = code[pre_rtarget]
 
             if code_pre_rtarget in self.jump_forward:
@@ -828,20 +833,86 @@ class Scanner2(scan.Scanner):
                             jump_target = self.get_target(next_offset, next_op)
                             if jump_target in self.setup_loops:
                                 self.structs.append({'type':  'while-loop',
-                                       'start': start - 3,
+                                       'start': jump_if_offset,
                                        'end':   jump_target})
-                                self.fixed_jumps[start-3] = jump_target
+                                self.fixed_jumps[jump_if_offset] = jump_target
                                 return
 
                 end = self.restrict_to_parent(if_end, parent)
 
-                self.structs.append({'type':  'if-then',
-                                       'start': start-3,
-                                       'end':   pre_rtarget})
+                if_then_maybe = None
+
+                if 2.2 <= self.version <= 2.6:
+                    # Take the JUMP_IF target. In an "if/then", it will be
+                    # a POP_TOP instruction and the instruction before it
+                    # will be a JUMP_FORWARD to just after the POP_TOP.
+                    # For example:
+                    # Good:
+                    # 3  JUMP_IF_FALSE        33  'to 39'
+                    # ..
+                    # 36  JUMP_FORWARD          1  'to 40'
+                    # 39  POP_TOP
+                    # 40 ...
+                    # example:
+
+                    # BAD (is an "and"):
+                    # 28  JUMP_IF_FALSE         4  'to 35'
+                    # ...
+                    # 32  JUMP_ABSOLUTE        40  'to 40' # should be 36 or there should
+                    #                                      # be a COME_FROM at the pop top
+                    #                                      # before 40 to 35
+                    # 35  POP_TOP
+                    # 36 ...
+                    # 39  POP_TOP
+                    # 39_0  COME_FROM 3
+                    # 40 ...
+
+                    if self.opc.opname[code[jump_if_offset]].startswith('JUMP_IF'):
+                        jump_if_target = code[jump_if_offset+1]
+                        if self.opc.opname[code[jump_if_target + jump_if_offset + 3]] == 'POP_TOP':
+                            jump_inst = jump_if_target + jump_if_offset
+                            jump_offset = code[jump_inst+1]
+                            jump_op = self.opc.opname[code[jump_inst]]
+                            if (jump_op == 'JUMP_FORWARD' and jump_offset == 1):
+                                self.structs.append({'type':  'if-then',
+                                                     'start': start-3,
+                                                     'end':   pre_rtarget})
+
+                                self.thens[start] = end
+                            elif jump_op == 'JUMP_ABSOLUTE':
+                                if_then_maybe = {'type':  'if-then',
+                                                 'start': start-3,
+                                                 'end':   pre_rtarget}
+
+                elif self.version == 2.7:
+                    self.structs.append({'type':  'if-then',
+                                         'start': start-3,
+                                         'end':   pre_rtarget})
 
                 self.not_continue.add(pre_rtarget)
 
                 if rtarget < end:
+                    # We have an "else" block  of some kind.
+                    # Is it associated with "if_then_maybe" seen above?
+                    # These will be linked in this funny way:
+
+                    # 198  JUMP_IF_FALSE        18  'to 219'
+                    # 201  POP_TOP
+                    # ...
+                    # 216  JUMP_ABSOLUTE       256  'to 256'
+                    # 219  POP_TOP
+                    # ...
+                    # 252  JUMP_FORWARD          1  'to 256'
+                    # 255  POP_TOP
+                    # 256
+                    if if_then_maybe and jump_op == 'JUMP_ABSOLUTE':
+                        jump_target = self.get_target(jump_inst, code[jump_inst])
+                        if self.opc.opname[code[end]] == 'JUMP_FORWARD':
+                            end_target = self.get_target(end, code[end])
+                            if jump_target == end_target:
+                                self.structs.append(if_then_maybe)
+                                self.thens[start] = end
+
                     self.structs.append({'type':  'else',
                                        'start': rtarget,
                                        'end':   end})
@@ -850,6 +921,7 @@ class Scanner2(scan.Scanner):
                     self.structs.append({'type':  'if-then',
                                            'start': start,
                                            'end':   rtarget})
+                    self.thens[start] = rtarget
                     if self.version == 2.7 or code[pre_rtarget+1] != self.opc.JUMP_FORWARD:
                         self.return_end_ifs.add(pre_rtarget)
 
@@ -883,11 +955,12 @@ class Scanner2(scan.Scanner):
         self.ignore_if = set()
         self.build_statement_indices()
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
         self.setup_loop_targets = {}  # target given setup_loop offset
         self.setup_loops = {}  # setup_loop offset given target
+        self.thens = {} # JUMP_IF's that separate the 'then' part of an 'if'
 
         targets = {}
         for offset in self.op_range(0, n):
@@ -895,7 +968,7 @@ class Scanner2(scan.Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, op)
+            self.detect_control_flow(offset, op)
 
             if op_has_argument(op, self.opc):
                 label = self.fixed_jumps.get(offset)

uncompyle6/scanners/scanner26.py

@@ -94,35 +94,32 @@ class Scanner26(scan.Scanner2):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # from xdis.bytecode import Bytecode
-        # bytecode = Bytecode(co, self.opc)
-        # for instr in bytecode.get_instructions(co):
-        #     print(instr._disassemble())
-
         # Container for tokens
         tokens = []
 
         customize = {}
+        if self.is_pypy:
+            customize['PyPy'] = 1
+
         Token = self.Token # shortcut
 
-        n = self.setup_code(co)
+        codelen = self.setup_code(co)
 
-        self.build_lines_data(co, n)
-        self.build_prev_op(n)
+        self.build_lines_data(co, codelen)
+        self.build_prev_op(codelen)
 
         free, names, varnames = self.unmangle_code_names(co, classname)
         self.names = names
 
-        codelen = len(self.code)
-
         # Scan for assertions. Later we will
         # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT'.
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
-        for i in self.op_range(0, n):
-            # We need to detect the difference between
-            # "raise AssertionError" and
-            # "assert"
+        for i in self.op_range(0, codelen):
+            # We need to detect the difference between:
+            #   raise AssertionError
+            #  and
+            #   assert ...
             if (self.code[i] == self.opc.JUMP_IF_TRUE and
                 i + 4 < codelen and
                 self.code[i+3] == self.opc.POP_TOP and
@@ -167,6 +164,11 @@ class Scanner26(scan.Scanner2):
                         offset="%s_%d" % (offset, jump_idx),
                         has_arg = True))
                     jump_idx += 1
+            elif offset in self.thens:
+                tokens.append(Token(
+                    'THEN', None, self.thens[offset],
+                    offset="%s_0" % offset,
+                    has_arg = True))
 
             has_arg = (op >= self.opc.HAVE_ARGUMENT)
             if has_arg:

uncompyle6/scanners/scanner3.py

@@ -148,7 +148,7 @@ class Scanner3(Scanner):
         """
 
         show_asm = self.show_asm if not show_asm else show_asm
-        # show_asm = 'after'
+        # show_asm = 'both'
         if show_asm in ('both', 'before'):
             bytecode = Bytecode(co, self.opc)
             for instr in bytecode.get_instructions(co):
@@ -199,6 +199,7 @@ class Scanner3(Scanner):
         # Get jump targets
         # Format: {target offset: [jump offsets]}
         jump_targets = self.find_jump_targets(show_asm)
+        last_op_was_break = False
 
         for inst in bytecode:
 
@@ -321,10 +322,12 @@ class Scanner3(Scanner):
                 if target <= inst.offset:
                     next_opname = self.opname[self.code[inst.offset+3]]
                     if (inst.offset in self.stmts and
-                        next_opname not in ('END_FINALLY', 'POP_BLOCK',
+                        (next_opname not in ('END_FINALLY', 'POP_BLOCK',
                                             # Python 3.0 only uses POP_TOP
                                             'POP_TOP')
-                        and inst.offset not in self.not_continue):
+                        and inst.offset not in self.not_continue) or
+                        (tokens[-1].type == 'RETURN_VALUE' and
+                         self.version < 3.5)):
                         opname = 'CONTINUE'
                     else:
                         opname = 'JUMP_BACK'
@@ -334,15 +337,21 @@ class Scanner3(Scanner):
                         # There are other situations where we don't catch
                         # CONTINUE as well.
                         if tokens[-1].type == 'JUMP_BACK' and tokens[-1].attr <= argval:
-                            # intern is used because we are changing the *previous* token
-                            tokens[-1].type = intern('CONTINUE')
-
+                            if tokens[-2].type == 'BREAK_LOOP':
+                                del tokens[-1]
+                            else:
+                                # intern is used because we are changing the *previous* token
+                                tokens[-1].type = intern('CONTINUE')
+                    if last_op_was_break and opname == 'CONTINUE':
+                        last_op_was_break = False
+                        continue
             elif op == self.opc.RETURN_VALUE:
                 if inst.offset in self.return_end_ifs:
                     opname = 'RETURN_END_IF'
             elif inst.offset in self.load_asserts:
                 opname = 'LOAD_ASSERT'
 
+            last_op_was_break = opname == 'BREAK_LOOP'
             tokens.append(
                 Token(
                     type_ = opname,
@@ -434,7 +443,7 @@ class Scanner3(Scanner):
         self.build_statement_indices()
         self.else_start = {}
 
-        # Containers filled by detect_structure()
+        # Containers filled by detect_control_flow()
         self.not_continue = set()
         self.return_end_ifs = set()
         self.setup_loop_targets = {}  # target given setup_loop offset
@@ -446,7 +455,7 @@ class Scanner3(Scanner):
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(offset, targets)
+            self.detect_control_flow(offset, targets)
 
             has_arg = (op >= op3.HAVE_ARGUMENT)
             if has_arg:
@@ -573,7 +582,7 @@ class Scanner3(Scanner):
 
         return target
 
-    def detect_structure(self, offset, targets):
+    def detect_control_flow(self, offset, targets):
         """
         Detect structures and their boundaries to fix optimized jumps
         in python2.3+
@@ -600,7 +609,6 @@ class Scanner3(Scanner):
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
-
             # We categorize loop types: 'for', 'while', 'while 1' with
             # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
@@ -618,20 +626,30 @@ class Scanner3(Scanner):
             jump_back = self.last_instr(start, end, self.opc.JUMP_ABSOLUTE,
                                           next_line_byte, False)
 
-            if jump_back and jump_back != self.prev_op[end] and self.is_jump_forward(jump_back+3):
-                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE
-                    or (code[self.prev_op[end]] == self.opc.POP_BLOCK
-                         and code[self.prev_op[self.prev_op[end]]] == self.opc.RETURN_VALUE)):
+            jump_forward_offset = jump_back+3
+            return_val_offset1 = self.prev[self.prev[end]]
+
+            if (jump_back and jump_back != self.prev_op[end]
+                and self.is_jump_forward(jump_forward_offset)):
+                if (code[self.prev_op[end]] == self.opc.RETURN_VALUE or
+                    (code[self.prev_op[end]] == self.opc.POP_BLOCK
+                     and code[return_val_offset1] == self.opc.RETURN_VALUE)):
                     jump_back = None
-            if not jump_back: # loop suite ends in return. wtf right?
+            if not jump_back:
+                # loop suite ends in return
                 jump_back = self.last_instr(start, end, self.opc.RETURN_VALUE) + 1
                 if not jump_back:
                     return
+
+                jump_back += 1
+                if_offset = None
                 if code[self.prev_op[next_line_byte]] not in POP_JUMP_TF:
-                    loop_type = 'for'
-                else:
+                    if_offset = self.prev[next_line_byte]
+                if if_offset:
                     loop_type = 'while'
-                    self.ignore_if.add(self.prev_op[next_line_byte])
+                    self.ignore_if.add(if_offset)
+                else:
+                    loop_type = 'for'
                 target = next_line_byte
                 end = jump_back + 3
             else:
@@ -645,6 +663,7 @@ class Scanner3(Scanner):
                 elif target < offset:
                     self.fixed_jumps[offset] = jump_back+4
                     end = jump_back+4
+
                 target = self.get_target(jump_back)
 
                 if code[target] in (self.opc.FOR_ITER, self.opc.GET_ITER):
@@ -652,6 +671,7 @@ class Scanner3(Scanner):
                 else:
                     loop_type = 'while'
                     test = self.prev_op[next_line_byte]
+
                     if test == offset:
                         loop_type = 'while 1'
                     elif self.code[test] in op3.hasjabs+op3.hasjrel:
@@ -692,38 +712,40 @@ class Scanner3(Scanner):
                                      'end': prev_op[target]})
                 return
 
-            # Is it an "and" inside an "if" block
+            # The op offset just before the target jump offset is important
+            # in making a determination of what we have. Save that.
+            pre_rtarget = prev_op[rtarget]
+
+            # Is it an "and" inside an "if" or "while" block
             if op == self.opc.POP_JUMP_IF_FALSE:
+
                 # Search for another POP_JUMP_IF_FALSE targetting the same op,
                 # in current statement, starting from current offset, and filter
                 # everything inside inner 'or' jumps and midline ifs
                 match = self.rem_or(start, self.next_stmt[offset],
                                     self.opc.POP_JUMP_IF_FALSE, target)
-                # We can't remove mid-line ifs because line structures have changed
-                # from restructBytecode().
-                #  match = self.remove_mid_line_ifs(match)
 
                 # If we still have any offsets in set, start working on it
                 if match:
-                    is_jump_forward = self.is_jump_forward(prev_op[rtarget])
-                    if (is_jump_forward and prev_op[rtarget] not in self.stmts and
-                        self.restrict_to_parent(self.get_target(prev_op[rtarget]), parent) == rtarget):
-                        if (code[prev_op[prev_op[rtarget]]] == self.opc.JUMP_ABSOLUTE
+                    is_jump_forward = self.is_jump_forward(pre_rtarget)
+                    if (is_jump_forward and pre_rtarget not in self.stmts and
+                        self.restrict_to_parent(self.get_target(pre_rtarget), parent) == rtarget):
+                        if (code[prev_op[pre_rtarget]] == self.opc.JUMP_ABSOLUTE
                             and self.remove_mid_line_ifs([offset]) and
-                            target == self.get_target(prev_op[prev_op[rtarget]]) and
-                            (prev_op[prev_op[rtarget]] not in self.stmts or
-                             self.get_target(prev_op[prev_op[rtarget]]) > prev_op[prev_op[rtarget]]) and
-                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], POP_JUMP_TF, target)))):
+                            target == self.get_target(prev_op[pre_rtarget]) and
+                            (prev_op[pre_rtarget] not in self.stmts or
+                             self.get_target(prev_op[pre_rtarget]) > prev_op[pre_rtarget]) and
+                            1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget], POP_JUMP_TF, target)))):
                             pass
-                        elif (code[prev_op[prev_op[rtarget]]] == self.opc.RETURN_VALUE
+                        elif (code[prev_op[pre_rtarget]] == self.opc.RETURN_VALUE
                               and self.remove_mid_line_ifs([offset]) and
-                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                              1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                                  POP_JUMP_TF, target))) |
-                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]],
+                                    set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[pre_rtarget],
                                                                              (self.opc.POP_JUMP_IF_FALSE,
                                                                               self.opc.POP_JUMP_IF_TRUE,
                                                                               self.opc.JUMP_ABSOLUTE),
-                                                                             prev_op[rtarget], True)))))):
+                                                                             pre_rtarget, True)))))):
                             pass
                         else:
                             fix = None
@@ -751,7 +773,7 @@ class Scanner3(Scanner):
                     if code[prev_op[next]] == self.opc.POP_JUMP_IF_FALSE:
                         if (code[next] == self.opc.JUMP_FORWARD
                             or target != rtarget
-                            or code[prev_op[prev_op[rtarget]]] not in
+                            or code[prev_op[pre_rtarget]] not in
                             (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE)):
                             self.fixed_jumps[offset] = prev_op[next]
                             return
@@ -764,14 +786,14 @@ class Scanner3(Scanner):
             if offset in self.ignore_if:
                 return
 
-            if (code[prev_op[rtarget]] == self.opc.JUMP_ABSOLUTE and
-                prev_op[rtarget] in self.stmts and
-                prev_op[rtarget] != offset and
-                prev_op[prev_op[rtarget]] != offset and
+            if (code[pre_rtarget] == self.opc.JUMP_ABSOLUTE and
+                pre_rtarget in self.stmts and
+                pre_rtarget != offset and
+                prev_op[pre_rtarget] != offset and
                 not (code[rtarget] == self.opc.JUMP_ABSOLUTE and
                      code[rtarget+3] == self.opc.POP_BLOCK and
-                     code[prev_op[prev_op[rtarget]]] != self.opc.JUMP_ABSOLUTE)):
-                rtarget = prev_op[rtarget]
+                     code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE)):
+                rtarget = pre_rtarget
 
             # Does the "jump if" jump beyond a jump op?
             # That is, we have something like:
@@ -787,12 +809,11 @@ class Scanner3(Scanner):
             # There are other contexts we may need to consider
             # like whether the target is "END_FINALLY"
             # or if the condition jump is to a forward location
-            if self.is_jump_forward(prev_op[rtarget]):
-                rrtarget = prev_op[rtarget]
-                if_end = self.get_target(rrtarget)
+            if self.is_jump_forward(pre_rtarget):
+                if_end = self.get_target(pre_rtarget)
 
                 # If the jump target is back, we are looping
-                if (if_end < rrtarget and
+                if (if_end < pre_rtarget and
                     (code[prev_op[if_end]] == self.opc.SETUP_LOOP)):
                     if (if_end > start):
                         return
@@ -801,19 +822,25 @@ class Scanner3(Scanner):
 
                 self.structs.append({'type': 'if-then',
                                      'start': start,
-                                     'end': prev_op[rtarget]})
-                self.not_continue.add(prev_op[rtarget])
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
 
                 if rtarget < end and (
                         code[rtarget] not in (self.opc.END_FINALLY,
                                               self.opc.JUMP_ABSOLUTE) and
-                        code[prev_op[rrtarget]] not in (self.opc.POP_EXCEPT,
+                        code[prev_op[pre_rtarget]] not in (self.opc.POP_EXCEPT,
                                                         self.opc.END_FINALLY)):
                     self.structs.append({'type': 'else',
                                          'start': rtarget,
                                          'end': end})
                     self.else_start[rtarget] = end
-            elif code[prev_op[rtarget]] == self.opc.RETURN_VALUE:
+            elif self.is_jump_back(pre_rtarget):
+                if_end = rtarget
+                self.structs.append({'type': 'if-then',
+                                     'start': start,
+                                     'end': pre_rtarget})
+                self.not_continue.add(pre_rtarget)
+            elif code[pre_rtarget] == self.opc.RETURN_VALUE:
                 self.structs.append({'type': 'if-then',
                                      'start': start,
                                      'end': rtarget})
@@ -843,7 +870,7 @@ class Scanner3(Scanner):
                                 return
                             pass
                     pass
-                self.return_end_ifs.add(prev_op[rtarget])
+                self.return_end_ifs.add(pre_rtarget)
 
         elif op in self.jump_if_pop:
             target = self.get_target(offset)
@@ -882,6 +909,16 @@ class Scanner3(Scanner):
                 pass
         return
 
+    def is_jump_back(self, offset):
+        """
+        Return True if the code at offset is some sort of jump back.
+        That is, it is ether "JUMP_FORWARD" or an absolute jump that
+        goes forward.
+        """
+        if self.code[offset] != self.opc.JUMP_ABSOLUTE:
+            return False
+        return offset > self.get_target(offset)
+
     def next_except_jump(self, start):
         """
         Return the next jump that was generated by an except SomeException:

uncompyle6/semantics/pysource.py

@@ -301,7 +301,7 @@ TABLE_DIRECT = {
     'ifstmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
     'iflaststmt':		( '%|if %c:\n%+%c%-', 0, 1 ),
     'iflaststmtl':		( '%|if %c:\n%+%c%-', 0, 1 ),
-    'testtrue':     ( 'not %p', (0, 22) ),
+    'testtrue':         ( 'not %p', (0, 22) ),
 
     'ifelsestmt':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
     'ifelsestmtc':	( '%|if %c:\n%+%c%-%|else:\n%+%c%-', 0, 1, 3 ),
@@ -589,7 +589,9 @@ class SourceWalker(GenericASTTraversal, object):
             })
         else:
             TABLE_DIRECT.update({
-                'except_cond3':	( '%|except %c, %c:\n', 1, 6 ),
+                'except_cond3':	 ( '%|except %c, %c:\n', 1, 6 ),
+                'testtrue_then': ( 'not %p', (0, 22) ),
+
             })
         if 2.4 <= version <= 2.6:
             TABLE_DIRECT.update({

uncompyle6/verify.py

@@ -407,7 +407,8 @@ def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False):
     try:
         code_obj2 = load_file(src_filename)
     except SyntaxError as e:
-        return str(e)
+        # src_filename can be the first of a group sometimes
+        return str(e).replace(src_filename, pyc_filename)
     cmp_code_objects(version, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify)
     return None
 

uncompyle6/version.py

@@ -1,3 +1,3 @@
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='2.9.8'
+VERSION='2.9.9'