Bug in 3.x detecting "if" structure and ...

scanner3.py: bug in 3.x detecting "if" structure
Make scanner2.py look more like scanner3.py
verify.py: add weak-verify which tests Pytyon syntax, but not code

test/simple_source/bug33/04_aug_assign.py

@@ -0,0 +1,11 @@
+# From python 3.4 difflib
+# Bug in 3.x is combining bestsize +=1 with while condition
+bestsize = 5
+b = [2]
+def isbjunk(x):
+    return False
+
+while 1 < bestsize and \
+      not isbjunk(b[0]) and \
+      b[0]:
+    bestsize += 1

test/test_pyenvlib.py

@@ -111,11 +111,13 @@ if __name__ == '__main__':
     test_options_keys = list(test_options.keys())
     test_options_keys.sort()
     opts, args = getopt.getopt(sys.argv[1:], '',
-                               ['start-with=', 'verify', 'all', ] \
+                               ['start-with=', 'verify', 'weak-verify', 'all', ] \
                                + test_options_keys )
     for opt, val in opts:
         if opt == '--verify':
             do_verify = True
+        if opt == '--weak-verify':
+            do_verify = 'weak'
         elif opt == '--start-with':
             start_with = val
         elif opt[2:] in test_options_keys:

uncompyle6/main.py

@@ -147,8 +147,9 @@ def main(in_base, out_base, files, codes, outfile=None,
             if outfile:
                 outstream.close()
                 if do_verify:
+                    weak_verify = do_verify == 'weak'
                     try:
-                        msg = verify.compare_code_with_srcfile(infile, outfile)
+                        msg = verify.compare_code_with_srcfile(infile, outfile, weak_verify=weak_verify)
                         if not outfile:
                             if not msg:
                                 print('\n# okay decompiling %s' % infile)

uncompyle6/scanners/scanner2.py

@@ -438,7 +438,7 @@ class Scanner2(scan.Scanner):
             elif op in self.setup_ops:
                 count_SETUP_ += 1
 
-    def detect_structure(self, pos, op=None):
+    def detect_structure(self, pos, op):
         '''
         Detect type of block structures and their boundaries to fix optimized jumps
         in python2.3+
@@ -447,26 +447,23 @@ class Scanner2(scan.Scanner):
         # TODO: check the struct boundaries more precisely -Dan
 
         code = self.code
-        # Ev remove this test and make op a mandatory argument -Dan
-        if op is None:
-            op = code[pos]
 
         # Detect parent structure
         parent = self.structs[0]
         start  = parent['start']
         end    = parent['end']
-        for s in self.structs:
+        for struct in self.structs:
-            _start = s['start']
+            _start = struct['start']
-            _end   = s['end']
+            _end   = struct['end']
             if (_start <= pos < _end) and (_start >= start and _end <= end):
                 start  = _start
                 end    = _end
-                parent = s
+                parent = struct
 
         if op == self.opc.SETUP_LOOP:
 
             # We categorize loop types: 'for', 'while', 'while 1' with
-            # possibly suffices '-loop' and '-else'
+            # possibly suffixes '-loop' and '-else'
             # Try to find the jump_back instruction of the loop.
             # It could be a return instruction.
 
@@ -803,30 +800,31 @@ class Scanner2(scan.Scanner):
         self.return_end_ifs = set()
 
         targets = {}
-        for i in self.op_range(0, n):
+        for offset in self.op_range(0, n):
-            op = self.code[i]
+            op = self.code[offset]
 
             # Determine structures and fix jumps in Python versions
             # since 2.3
-            self.detect_structure(i, op)
+            self.detect_structure(offset, op)
 
             if op >= self.opc.HAVE_ARGUMENT:
-                label = self.fixed_jumps.get(i)
+                label = self.fixed_jumps.get(offset)
-                oparg = self.get_argument(i)
+                oparg = self.get_argument(offset)
+
                 if label is None:
                     if op in self.opc.hasjrel and op != self.opc.FOR_ITER:
-                        label = i + 3 + oparg
+                        label = offset + 3 + oparg
                     elif self.version == 2.7 and op in self.opc.hasjabs:
                         if op in (self.opc.JUMP_IF_FALSE_OR_POP,
                                   self.opc.JUMP_IF_TRUE_OR_POP):
-                            if (oparg > i):
+                            if (oparg > offset):
                                 label = oparg
 
                 if label is not None and label != -1:
-                    targets[label] = targets.get(label, []) + [i]
+                    targets[label] = targets.get(label, []) + [offset]
-            elif op == self.opc.END_FINALLY and i in self.fixed_jumps:
+            elif op == self.opc.END_FINALLY and offset in self.fixed_jumps:
-                label = self.fixed_jumps[i]
+                label = self.fixed_jumps[offset]
-                targets[label] = targets.get(label, []) + [i]
+                targets[label] = targets.get(label, []) + [offset]
         return targets
 
     # FIXME: combine with scanner3.py code and put into scanner.py

uncompyle6/scanners/scanner3.py

@@ -415,7 +415,7 @@ class Scanner3(Scanner):
 
                 if label is None:
                     if op in op3.hasjrel and op != self.opc.FOR_ITER:
-                        label = offset + self.op_size(op) + oparg
+                        label = offset + 3 + oparg
                     elif op in op3.hasjabs:
                         if op in self.jump_if_pop:
                             if oparg > offset:
@@ -547,6 +547,12 @@ class Scanner3(Scanner):
                 parent = struct
 
         if op == self.opc.SETUP_LOOP:
+
+            # We categorize loop types: 'for', 'while', 'while 1' with
+            # possibly suffixes '-loop' and '-else'
+            # Try to find the jump_back instruction of the loop.
+            # It could be a return instruction.
+
             start = offset+3
             target = self.get_target(offset)
             end    = self.restrict_to_parent(target, parent)
@@ -679,8 +685,7 @@ class Scanner3(Scanner):
                             self.fixed_jumps[offset] = fix or match[-1]
                             return
                     else:
-                        if is_jump_forward:
+                        self.fixed_jumps[offset] = match[-1]
-                            self.fixed_jumps[offset] = match[-1]
                         return
             # op == POP_JUMP_IF_TRUE
             else:

uncompyle6/verify.py

@@ -133,7 +133,8 @@ class CmpErrorMember(VerifyCmpError):
 # these members are ignored
 __IGNORE_CODE_MEMBERS__ = ['co_filename', 'co_firstlineno', 'co_lnotab', 'co_stacksize', 'co_names']
 
-def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, name=''):
+def cmp_code_objects(version, is_pypy, code_obj1, code_obj2,
+                     name='', ignore_code=False):
     """
     Compare two code-objects.
 
@@ -178,9 +179,9 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, name=''):
 
     tokens1 = None
     for member in members:
-        if member in __IGNORE_CODE_MEMBERS__:
+        if member in __IGNORE_CODE_MEMBERS__ or ignore_code:
             pass
-        elif member == 'co_code':
+        elif member == 'co_code' and not ignore_code:
             if version == 2.3:
                 import uncompyle6.scanners.scanner23 as scan
                 scanner = scan.Scanner26()
@@ -379,7 +380,7 @@ class Token(scanner.Token):
     def __str__(self):
         return '%s\t%-17s %r' % (self.offset, self.type, self.pattr)
 
-def compare_code_with_srcfile(pyc_filename, src_filename):
+def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False):
     """Compare a .pyc with a source code file."""
     version, timestamp, magic_int, code_obj1, is_pypy = load_module(pyc_filename)
     if magic_int != PYTHON_MAGIC_INT:
@@ -387,14 +388,14 @@ def compare_code_with_srcfile(pyc_filename, src_filename):
                % (PYTHON_MAGIC_INT, magic_int))
         return msg
     code_obj2 = load_file(src_filename)
-    cmp_code_objects(version, is_pypy, code_obj1, code_obj2)
+    cmp_code_objects(version, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify)
     return None
 
-def compare_files(pyc_filename1, pyc_filename2):
+def compare_files(pyc_filename1, pyc_filename2, weak_verify=False):
     """Compare two .pyc files."""
     version, timestamp, magic_int1, code_obj1, is_pypy = uncompyle6.load_module(pyc_filename1)
     version, timestamp, magic_int2, code_obj2, is_pypy = uncompyle6.load_module(pyc_filename2)
-    cmp_code_objects(version, is_pypy, code_obj1, code_obj2)
+    cmp_code_objects(version, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify)
 
 if __name__ == '__main__':
     t1 = Token('LOAD_CONST', None, 'code_object _expandLang', 52)