Stronger while 1 testing...

Make instructions available in reduce tests.
Back off of a while test that was semantically different.

pytest/test_pysource.py

@@ -1,4 +1,6 @@
+import sys
 from uncompyle6 import PYTHON3
+from uncompyle6.scanner import get_scanner
 from uncompyle6.semantics.consts import (
     escape, NONE,
     # RETURN_NONE, PASS, RETURN_LOCALS
@@ -17,7 +19,10 @@ from uncompyle6.semantics.pysource import SourceWalker as SourceWalker
 
 def test_template_engine():
     s = StringIO()
-    sw = SourceWalker(2.7, s, None)
+    sys_version = float(sys.version[0:3])
+    scanner = get_scanner(sys_version, is_pypy=False)
+    scanner.insts = []
+    sw = SourceWalker(2.7, s, scanner)
     sw.ast = NONE
     sw.template_engine(('--%c--', 0), NONE)
     print(sw.f.getvalue())

test/stdlib/runtests.sh

@@ -41,6 +41,7 @@ case $PYVERSION in
 	    [test_grp.py]=1      # Long test - might work Control flow?
 	    [test_imp.py]=1
 	    [test_math.py]=1 # Control flow?
+	    [test_pdb.py]=1
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
 	    [test_queue.py]=1 # Control flow?
 	    [test_re.py]=1 # Probably Control flow?
@@ -135,7 +136,8 @@ else
     files=test_*.py
 fi
 for file in $files; do
-    [[ -v SKIP_TESTS[$file] ]] && continue
+    # AIX bash doesn't grok [[ -v SKIP... ]]
+    [[ ${SKIP_TESTS[$file]} == 1 ]] && continue
 
     # If the fails *before* decompiling, skip it!
     typeset -i STARTTIME=$(date +%s)

uncompyle6/parser.py

@@ -65,6 +65,8 @@ class PythonParser(GenericASTBuilder):
         # semantic actions
         self.singleton = frozenset(('str', 'joined_str', 'store', '_stmts', 'suite_stmts_opt',
                                     'inplace_op'))
+        # Instructions filled in from scanner
+        self.insts = []
 
     def ast_first_offset(self, ast):
         if hasattr(ast, 'offset'):

uncompyle6/parsers/parse3.py

@@ -358,9 +358,8 @@ class Python3Parser(PythonParser):
                               COME_FROM_LOOP
 
         # FIXME: Python 3.? starts adding branch optimization? Put this starting there.
-        while1stmt        ::= SETUP_LOOP l_stmts
-        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
 
+        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
         while1stmt        ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
 
         while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK
@@ -1056,9 +1055,29 @@ class Python3Parser(PythonParser):
                 last += 1
             return tokens[first].attr == tokens[last].offset
         elif lhs == 'while1stmt':
+
+            # If there is a fall through to the COME_FROM_LOOP. then this is
+            # not a while 1. So the instruction before should either be a
+            # JUMP_BACK or the instruction before should not be the target of a
+            # jump. (Well that last clause i not quite right; that target could be
+            # from dead code. Ugh. We need a more uniform control flow analysis.)
+            if last == len(tokens) or tokens[last-1] == 'COME_FROM_LOOP':
+                cfl = last-1
+            else:
+                cfl = last
+            assert tokens[cfl] == 'COME_FROM_LOOP'
+
+            if tokens[cfl-1] != 'JUMP_BACK':
+                cfl_offset = tokens[cfl-1].offset
+                insn = next(i for i in self.insts if cfl_offset == i.offset)
+                if insn and  insn.is_jump_target:
+                    return True
+
+            # Check that the SETUP_LOOP jumps to the offset after the
+            # COME_FROM_LOOP
             if (0 <= last < len(tokens)
                 and tokens[last] in ('COME_FROM_LOOP', 'JUMP_BACK')):
-                # jump_back should be right afer SETUP_LOOP. Test?
+                # jump_back should be right before COME_FROM_LOOP?
                 last += 1
             while last < len(tokens) and isinstance(tokens[last].offset, str):
                 last += 1

uncompyle6/semantics/pysource.py

@@ -213,6 +213,9 @@ class SourceWalker(GenericASTTraversal, object):
         self.linestarts = linestarts
         self.line_number = 0
         self.ast_errors = []
+        # FIXME: have p.insts update in a better way
+        # modularity is broken here
+        self.p.insts = scanner.insts
 
         # This is in Python 2.6 on. It changes the way
         # strings get interpreted. See n_LOAD_CONST
@@ -2508,7 +2511,12 @@ class SourceWalker(GenericASTTraversal, object):
                     t.kind = 'RETURN_VALUE_LAMBDA'
             tokens.append(Token('LAMBDA_MARKER'))
             try:
+                # FIXME: have p.insts update in a better way
+                # modularity is broken here
+                p_insts = self.p.insts
+                self.p.insts = self.scanner.insts
                 ast = python_parser.parse(self.p, tokens, customize)
+                self.p.insts = p_insts
             except (python_parser.ParserError, AssertionError) as e:
                 raise ParserError(e, tokens)
             maybe_show_ast(self.showast, ast)
@@ -2535,7 +2543,12 @@ class SourceWalker(GenericASTTraversal, object):
 
         # Build AST from disassembly.
         try:
+            # FIXME: have p.insts update in a better way
+            # modularity is broken here
+            p_insts = self.p.insts
+            self.p.insts = self.scanner.insts
             ast = python_parser.parse(self.p, tokens, customize)
+            self.p.insts = p_insts
         except (python_parser.ParserError, AssertionError) as e:
             raise ParserError(e, tokens)
 
@@ -2560,7 +2573,8 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
     # store final output stream for case of error
     scanner = get_scanner(version, is_pypy=is_pypy)
 
-    tokens, customize = scanner.ingest(co, code_objects=code_objects, show_asm=showasm)
+    tokens, customize = scanner.ingest(co, code_objects=code_objects,
+                                       show_asm=showasm)
 
     debug_parser = dict(PARSER_DEFAULT_DEBUG)
     if showgrammar:
@@ -2583,7 +2597,8 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
 
     assert deparsed.ast == 'stmts', 'Should have parsed grammar start'
 
-    del tokens # save memory
+    # save memory
+    del tokens
 
     deparsed.mod_globs = find_globals(deparsed.ast, set())