diff --git a/test/Makefile b/test/Makefile index b6d45f90..70e01d1f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -171,10 +171,12 @@ grammar-coverage-3.5: #: Check deparsing Python 2.6 check-bytecode-2.6: $(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify + $(PYTHON) test_pythonlib.py --bytecode-2.6-run --verify-run #: Check deparsing Python 2.7 check-bytecode-2.7: $(PYTHON) test_pythonlib.py --bytecode-2.7 --weak-verify + $(PYTHON) test_pythonlib.py --bytecode-2.7-run --verify-run #: Check deparsing Python 3.0 check-bytecode-3.0: @@ -191,22 +193,27 @@ check-bytecode-3.2: #: Check deparsing Python 3.3 check-bytecode-3.3: $(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify + $(PYTHON) test_pythonlib.py --bytecode-3.3-run --verify-run #: Check deparsing Python 3.4 check-bytecode-3.4: $(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify + $(PYTHON) test_pythonlib.py --bytecode-3.4-run --verify-run #: Check deparsing Python 3.5 check-bytecode-3.5: $(PYTHON) test_pythonlib.py --bytecode-3.5 --weak-verify + $(PYTHON) test_pythonlib.py --bytecode-3.5-run --verify-run #: Check deparsing Python 3.6 check-bytecode-3.6: $(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify + $(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run #: short tests for bytecodes only for this version of Python check-native-short: $(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --weak-verify $(COMPILE) + $(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION)-run --verify-run $(COMPILE) #: Run longer Python 2.6's lib files known to be okay check-2.6-ok: diff --git a/test/bytecode_2.4_run/README b/test/bytecode_2.4_run/README new file mode 100644 index 00000000..3fa5e010 --- /dev/null +++ b/test/bytecode_2.4_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 2.4 + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 2.4 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_2.5_run/README b/test/bytecode_2.5_run/README new file mode 100644 index 00000000..097cc5ba --- /dev/null +++ b/test/bytecode_2.5_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 2.5. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 2.5 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_2.6_run/README b/test/bytecode_2.6_run/README new file mode 100644 index 00000000..a827636a --- /dev/null +++ b/test/bytecode_2.6_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 2.6. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 2.6 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_2.7_run/README b/test/bytecode_2.7_run/README new file mode 100644 index 00000000..50815908 --- /dev/null +++ b/test/bytecode_2.7_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 2.7. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 2.7 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_3.0/README b/test/bytecode_3.0/README new file mode 100644 index 00000000..bb6f05ca --- /dev/null +++ b/test/bytecode_3.0/README @@ -0,0 +1 @@ +These are byte-compiled programs compiled by Python 3.0 diff --git a/test/bytecode_3.0_run/README b/test/bytecode_3.0_run/README new file mode 100644 index 00000000..c89d5d61 --- /dev/null +++ b/test/bytecode_3.0_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 3.0. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 3.0 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_3.1/README b/test/bytecode_3.1/README new file mode 100644 index 00000000..cb4960b3 --- /dev/null +++ b/test/bytecode_3.1/README @@ -0,0 +1 @@ +These are byte-compiled programs compiled by Python 3.1 diff --git a/test/bytecode_3.1_run/README b/test/bytecode_3.1_run/README new file mode 100644 index 00000000..96ace639 --- /dev/null +++ b/test/bytecode_3.1_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 3.1. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 3.1 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_3.2_run/README b/test/bytecode_3.2_run/README new file mode 100644 index 00000000..7f82a6f6 --- /dev/null +++ b/test/bytecode_3.2_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 3.2. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 3.2 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_3.3_run/README b/test/bytecode_3.3_run/README new file mode 100644 index 00000000..68c21035 --- /dev/null +++ b/test/bytecode_3.3_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 3.3. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 3.3 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_3.4_run/README b/test/bytecode_3.4_run/README new file mode 100644 index 00000000..e60f8376 --- /dev/null +++ b/test/bytecode_3.4_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 3.4. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 3.4 interpreter, they will give an error if they +are miscompiled. diff --git a/test/bytecode_3.5_run/README b/test/bytecode_3.5_run/README new file mode 100644 index 00000000..a5f9700b --- /dev/null +++ b/test/bytecode_3.5_run/README @@ -0,0 +1,5 @@ +These are byte-compiled programs compiled by Python 3.5. + +Furthrmore the programs here are self-checking: when decompiled and +then run again in a 3.5 interpreter, they will give an error if they +are miscompiled. diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index 206141aa..6fa181e5 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -129,17 +129,19 @@ if __name__ == '__main__': test_options_keys = list(test_options.keys()) test_options_keys.sort() opts, args = getopt.getopt(sys.argv[1:], '', - ['start-with=', 'verify', 'weak-verify', + ['start-with=', 'verify', 'verify-run', 'weak-verify', 'max=', 'coverage', 'all', ] \ + test_options_keys ) vers = '' for opt, val in opts: if opt == '--verify': - do_verify = True - if opt == '--weak-verify': + do_verify = 'strong' + elif opt == '--weak-verify': do_verify = 'weak' - if opt == '--coverage': + elif opt == '--verify-run': + do_verify = 'verify-run' + elif opt == '--coverage': do_coverage = True elif opt == '--start-with': start_with = val diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index 384574f8..1b21153b 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -85,6 +85,9 @@ for vers in (1.5, bytecode = "bytecode_%s" % vers key = "bytecode-%s" % vers test_options[key] = (bytecode, PYC, bytecode, vers) + bytecode = "bytecode_%s_run" % vers + key = "bytecode-%s-run" % vers + test_options[key] = (bytecode, PYC, bytecode, vers) key = "%s" % vers pythonlib = "python%s" % vers if isinstance(vers, float) and vers >= 3.0: @@ -189,8 +192,9 @@ if __name__ == '__main__': test_options_keys = list(test_options.keys()) test_options_keys.sort() opts, args = getopt.getopt(sys.argv[1:], '', - ['start-with=', 'verify', 'weak-verify', 'all', 'compile', - 'coverage', + ['start-with=', 'verify', 'verify-run', + 'weak-verify', 'all', + 'compile', 'coverage', 'no-rm'] \ + test_options_keys ) if not opts: help() @@ -205,9 +209,11 @@ if __name__ == '__main__': for opt, val in opts: if opt == '--verify': - test_opts['do_verify'] = True + test_opts['do_verify'] = 'strong' elif opt == '--weak-verify': test_opts['do_verify'] = 'weak' + elif opt == '--verify-run': + test_opts['do_verify'] = 'verify-run' elif opt == '--compile': test_opts['do_compile'] = True elif opt == '--start-with': diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index 403998cd..596e925a 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -35,6 +35,7 @@ Options: -p use number of processes -r recurse directories looking for .pyc and .pyo files --verify compare generated source with input byte-code + --verify-run compile generated source, run it and check exit code --weak-verify compile generated source --linemaps generated line number correspondencies between byte-code and generated source output @@ -84,7 +85,8 @@ def main_bin(): try: opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:', 'help asm grammar linemaps recurse timestamp tree ' - 'verify version weak-verify showgrammar'.split(' ')) + 'verify verify-run version weak-verify ' + 'showgrammar'.split(' ')) except getopt.GetoptError as e: print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) @@ -101,6 +103,8 @@ def main_bin(): options['do_verify'] = 'strong' elif opt == '--weak-verify': options['do_verify'] = 'weak' + elif opt == '--verify-run': + options['do_verify'] = 'verify-run' elif opt == '--linemaps': options['do_linemaps'] = True elif opt in ('--asm', '-a'): diff --git a/uncompyle6/main.py b/uncompyle6/main.py index 3f39838d..8f8f6d59 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -215,9 +215,10 @@ def main(in_base, out_base, files, codes, outfile=None, outstream.close() if do_verify: - weak_verify = do_verify == 'weak' try: - msg = verify.compare_code_with_srcfile(infile, current_outfile, weak_verify=weak_verify) + msg = verify.compare_code_with_srcfile(infile, + current_outfile, + do_verify) if not current_outfile: if not msg: print('\n# okay decompiling %s' % infile) @@ -245,7 +246,6 @@ def main(in_base, out_base, files, codes, outfile=None, okay_files += 1 pass elif do_verify: - from trepan.api import debug; debug() sys.stderr.write("\n### uncompile successful, but no file to compare against\n") pass else: @@ -280,19 +280,21 @@ else: def status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files, weak_verify): if weak_verify == 'weak': - verification_type = 'weak' + verification_type = 'weak ' + elif weak_verify == 'verify-run': + verification_type = 'run ' else: - verification_type = 'strong' + verification_type = '' if tot_files == 1: if failed_files: return "\n# decompile failed" elif verify_failed_files: - return "\n# decompile %s verification failed" % verification_type + return "\n# decompile %sverification failed" % verification_type else: return "\n# Successfully decompiled file" pass pass mess = "decompiled %i files: %i okay, %i failed" % (tot_files, okay_files, failed_files) if do_verify: - mess += (", %i %s verification failed" % (verify_failed_files, verification_type)) + mess += (", %i %sverification failed" % (verify_failed_files, verification_type)) return mess diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index b0482d55..3480e111 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -1,6 +1,6 @@ # +# (C) Copyright 2015-2018 by Rocky Bernstein # (C) Copyright 2000-2002 by hartmut Goebel -# (C) Copyright 2015-2017 by Rocky Bernstein # """ byte-code verification @@ -8,11 +8,12 @@ byte-code verification from __future__ import print_function -import operator +import operator, sys import xdis.std as dis +from subprocess import call import uncompyle6 -import uncompyle6.scanner as scanner +from uncompyle6.scanner import (Token as ScannerToken, get_scanner) from uncompyle6 import PYTHON3 from xdis.code import iscode from xdis.magics import PYTHON_MAGIC_INT @@ -134,8 +135,8 @@ class CmpErrorMember(VerifyCmpError): # these members are ignored __IGNORE_CODE_MEMBERS__ = ['co_filename', 'co_firstlineno', 'co_lnotab', 'co_stacksize', 'co_names'] -def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, - name='', ignore_code=False): +def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, verify, + name=''): """ Compare two code-objects. @@ -180,53 +181,12 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, tokens1 = None for member in members: - if member in __IGNORE_CODE_MEMBERS__ or ignore_code: + if member in __IGNORE_CODE_MEMBERS__ or verify != 'verify': pass - elif member == 'co_code' and not ignore_code: - if version == 2.3: - import uncompyle6.scanners.scanner23 as scan - scanner = scan.Scanner23(show_asm=False) - elif version == 2.4: - import uncompyle6.scanners.scanner24 as scan - scanner = scan.Scanner24(show_asm=False) - elif version == 2.5: - import uncompyle6.scanners.scanner25 as scan - scanner = scan.Scanner25(show_asm=False) - elif version == 2.6: - import uncompyle6.scanners.scanner26 as scan - scanner = scan.Scanner26(show_asm=False) - elif version == 2.7: - if is_pypy: - import uncompyle6.scanners.pypy27 as scan - scanner = scan.ScannerPyPy27(show_asm=False) - else: - import uncompyle6.scanners.scanner27 as scan - scanner = scan.Scanner27() - elif version == 3.0: - import uncompyle6.scanners.scanner30 as scan - scanner = scan.Scanner30() - elif version == 3.1: - import uncompyle6.scanners.scanner32 as scan - scanner = scan.Scanner32() - elif version == 3.2: - if is_pypy: - import uncompyle6.scanners.pypy32 as scan - scanner = scan.ScannerPyPy32() - else: - import uncompyle6.scanners.scanner32 as scan - scanner = scan.Scanner32() - elif version == 3.3: - import uncompyle6.scanners.scanner33 as scan - scanner = scan.Scanner33() - elif version == 3.4: - import uncompyle6.scanners.scanner34 as scan - scanner = scan.Scanner34() - elif version == 3.5: - import uncompyle6.scanners.scanner35 as scan - scanner = scan.Scanner35() - elif version == 3.6: - import uncompyle6.scanners.scanner36 as scan - scanner = scan.Scanner36() + elif member == 'co_code': + if verify != 'strong': + continue + scanner = get_scanner(version, is_pypy, show_asm=False) global JUMP_OPS JUMP_OPS = list(scan.JUMP_OPS) + ['JUMP_BACK'] @@ -367,7 +327,8 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, codes2 = ( c for c in code_obj2.co_consts if hasattr(c, 'co_consts') ) for c1, c2 in zip(codes1, codes2): - cmp_code_objects(version, is_pypy, c1, c2, name=name) + cmp_code_objects(version, is_pypy, c1, c2, verify, + name=name) elif member == 'co_flags': flags1 = code_obj1.co_flags flags2 = code_obj2.co_flags @@ -388,7 +349,7 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, getattr(code_obj1, member), getattr(code_obj2, member)) -class Token(scanner.Token): +class Token(ScannerToken): """Token class with changed semantics for 'cmp()'.""" def __cmp__(self, o): t = self.kind # shortcut @@ -414,8 +375,10 @@ class Token(scanner.Token): def __str__(self): return '%s\t%-17s %r' % (self.offset, self.kind, self.pattr) -def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False): - """Compare a .pyc with a source code file.""" +def compare_code_with_srcfile(pyc_filename, src_filename, verify): + """Compare a .pyc with a source code file. If everything is okay, None + is returned. Otherwise a string message describing the mismatch is returned. + """ (version, timestamp, magic_int, code_obj1, is_pypy, source_size) = load_module(pyc_filename) if magic_int != PYTHON_MAGIC_INT: @@ -427,17 +390,27 @@ def compare_code_with_srcfile(pyc_filename, src_filename, weak_verify=False): except SyntaxError as e: # src_filename can be the first of a group sometimes return str(e).replace(src_filename, pyc_filename) - cmp_code_objects(version, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify) + cmp_code_objects(version, is_pypy, code_obj1, code_obj2, verify) + if verify == 'verify-run': + try: + retcode = call("%s %s" % (sys.executable, src_filename), shell=True) + if retcode != 0: + return "Child was terminated by signal %d" % retcode + pass + except OSError as e: + return "Execution failed: %s" % e + pass return None -def compare_files(pyc_filename1, pyc_filename2, weak_verify=False): +def compare_files(pyc_filename1, pyc_filename2, verify): """Compare two .pyc files.""" (version1, timestamp, magic_int1, code_obj1, is_pypy, source_size) = uncompyle6.load_module(pyc_filename1) (version2, timestamp, magic_int2, code_obj2, is_pypy, - source_size) = uncompyle6.load_module(pyc_filename2) - weak_verify = weak_verify or (magic_int1 != magic_int2) - cmp_code_objects(version1, is_pypy, code_obj1, code_obj2, ignore_code=weak_verify) + source_size) = uncompyle6.load_module(pyc_filename2) + if (magic_int1 != magic_int2) and verify == 'verify': + verify = 'weak_verify' + cmp_code_objects(version1, is_pypy, code_obj1, code_obj2, verify) if __name__ == '__main__': t1 = Token('LOAD_CONST', None, 'code_object _expandLang', 52)