From 8c879c02de15760bd2e1901d537a60bf1fe60b98 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 04:09:44 -0500 Subject: [PATCH 01/10] Small logic correction in run-time test. --- .../bytecode_2.7_run/15_mixed_expressions.pyc | Bin 1188 -> 1152 bytes .../bytecode_3.0_run/15_mixed_expressions.pyc | Bin 1233 -> 1195 bytes .../bytecode_3.1_run/15_mixed_expressions.pyc | Bin 1195 -> 1159 bytes .../expression/15_mixed_expressions.py | 1 - 4 files changed, 1 deletion(-) diff --git a/test/bytecode_2.7_run/15_mixed_expressions.pyc b/test/bytecode_2.7_run/15_mixed_expressions.pyc index 73a159970953fa0f92fecedc99a30ff39363d703..58990b1f5a6c59ff880b1026bcf442989d4e4fed 100644 GIT binary patch delta 342 zcmZ3&*}y5y{F#^Q2;aNdWCkc;2GR}+3=G9l6Qz&orARTPvNNRcGBh(XWHT`o-)1Ob zV`ySzNa16M;$%qWWJuv>5Ka*Q%YA21Fp~L2h`M+WoV94lZ=A~^GV0L2^@c{A!n0OdvSlAfF7-blF7*&}j=df4+0A=k# AjQ{`u delta 323 zcmZqRT*4{M{F#?4T>WxvG6NJa18D~Z28QCAiPFacQaKn>*%?xJ7@8RwQrH&GE$)US$HHI_m5Z4i$Ry2JF*%nhU#OS` z=y%=BytMQ=r3@W5Pt5;qe#hj-DCPwe7GUCGlwo0G6l0WOI~adF*5`3oE=k_@Tr3@JRDr5QPxgwp4fGIZEH sG5=S}0#u=!nU}WNl-Z9_!~@6^VB%qvVPRvGV3cFzWz=ArJd?!&0390>dH?_b delta 94 zcmZ3@d684znunJwT>WxvG6ND|1+pE0xOmY-`3oGW91N-K3@O|bSIbO%&(6rb@jC~T mP$>&gUNb(M3>Mu0 diff --git a/test/bytecode_3.1_run/15_mixed_expressions.pyc b/test/bytecode_3.1_run/15_mixed_expressions.pyc index e64decfc714bf7cb13ce71cdd5764c42752bcc16..5e0693ce0fca87ec2788c7382e76756c5beb2834 100644 GIT binary patch delta 355 zcmZ3@+0H5N&%?`ggzsH!G6ND|1+pE0xHxK}{4viINrqH*h7=x#W=4i=CWhkM3?*y~ zO^ggFybMvC45^$9DSQmVDg0o$?@&3Ar~rdAR9*@!CpT57l z!Dd)P0~`d z>H0vJ3KoVGVTNp0hN5|lDIyF}tPH8FAd{q1M8T>nq3VFfz~yA2EHSW(c~BL33?O|# zsT3iQ8B#zF$eFuf28bgJkcpanf{{n|4onKkY_Sw(u(ltQKQc-ii!ed`0d|ue&}5L2 zT1cV-4Bk-JNPy+sCfhTK%Eln6U}o@vs(|ROm|Vz|B~;1+40_$nytMQ=r3@W5Pt5;q je#_*?DCPwe7GUCGlwo0G6l0WO 2.4: assert str(float("inf") * 0.0) == "nan" else: assert str(float("inf") * 0.0) == "-nan" -assert str(float("-inf") * 0.0) == "nan" assert -1e300 * 1e300 == float("-inf") # Complex (adapted from 02_complex.py) From 99b8a99ffaf71a7e76fab0775d9563c590705587 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 06:57:02 -0500 Subject: [PATCH 02/10] Python 2.5 fixes.. * "with" handling. * Go over 2.5 runtest.sh exclusions --- .../expression/15_mixed_expressions.py | 2 +- test/stdlib/runtests.sh | 22 ++++++++++++++++--- uncompyle6/parsers/parse25.py | 12 +++++++--- uncompyle6/semantics/customize.py | 4 +++- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/test/simple_source/expression/15_mixed_expressions.py b/test/simple_source/expression/15_mixed_expressions.py index 641b67b7..29995f1a 100644 --- a/test/simple_source/expression/15_mixed_expressions.py +++ b/test/simple_source/expression/15_mixed_expressions.py @@ -11,7 +11,7 @@ PYTHON_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0) x = 1e300 assert 0.0 == x * 0 assert x * 1e300 == float("inf") -if PYTHON_VERSION > 2.4: +if PYTHON_VERSION > 2.41: assert str(float("inf") * 0.0) == "nan" else: assert str(float("inf") * 0.0) == "-nan" diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 62e8db59..acb1cb92 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -34,10 +34,17 @@ case $PYVERSION in SKIP_TESTS=( [test_dis.py]=1 # We change line numbers - duh! [test_grp.py]=1 # Long test - might work Control flow? - [test_pwd.py]=1 # Long test - might work? Control flow? [test_pep247.py]=1 # Long test - might work? Control flow? + [test_pwd.py]=1 # Long test - might work? Control flow? + [test_pyclbr.py]=1 # Bug in checkModule()/ismethod() try confused for try-else + [test_pyexpat.py]=1 # Investigate [test_queue.py]=1 # Control flow? - # [test_threading.py]=1 # Long test - works + [test_re.py]=1 # try confused with try-else again + [test_socketserver.py]=1 # -- test takes too long to run: 40 seconds + [test_threading.py]=1 # Line numbers are expected to be different + [test_thread.py]=1 # test takes too long to run: 36 seconds + [test_trace.py]=1 # Long test - works + [test_zipfile64.py]=1 # Runs ok but takes 204 seconds ) ;; 2.5) @@ -47,9 +54,17 @@ case $PYVERSION in [test_grammar.py]=1 # Too many stmts. Handle large stmts [test_grp.py]=1 # Long test - might work Control flow? [test_pdb.py]=1 # Line-number specific + [test_pep247.py]=1 # "assert xxx or .." not detected properly in check_hash_module() + [test_pep352.py]=1 # try confused with try-else again; in test_inheritance() [test_pwd.py]=1 # Long test - might work? Control flow? + [test_pyclbr.py]=1 # Bug in checkModule()/ismethod() try confused for try-else [test_queue.py]=1 # Control flow? - [test_re.py]=1 # Probably Control flow? + [test_re.py]=1 # Possibly try confused with try-else again + [test_struct.py]=1 # "if and" confused for if .. assert and + [test_sys.py]=1 # try confused with try-else again; in test_current_frames() + [test_tarfile.py]=1 # try confused with try-else again; top-level import + [test_threading.py]=1 # Line numbers are expected to be different + [test_thread.py]=1 # test takes too long to run: 36 seconds [test_trace.py]=1 # Line numbers are expected to be different [test_zipfile64.py]=1 # Runs ok but takes 204 seconds ) @@ -94,6 +109,7 @@ case $PYVERSION in [test_re.py]=1 # Probably Control flow? [test_queue.py]=1 # Control flow? [test_trace.py]=1 # Line numbers are expected to be different + [test_urllib2net.py]=1 # Fails on its own. May need interactive input [test_zipfile64.py]=1 # Skip Long test [test_zlib.py]=1 # Takes too long to run (more than 3 minutes 39 seconds) # .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc diff --git a/uncompyle6/parsers/parse25.py b/uncompyle6/parsers/parse25.py index ef2447a2..5b1740ae 100644 --- a/uncompyle6/parsers/parse25.py +++ b/uncompyle6/parsers/parse25.py @@ -25,11 +25,17 @@ class Python25Parser(Python26Parser): setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 setup_finally # opcode SETUP_WITH - setupwith ::= DUP_TOP LOAD_ATTR STORE_NAME LOAD_ATTR CALL_FUNCTION_0 POP_TOP - withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM with_cleanup + setupwith ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0 POP_TOP + withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM with_cleanup + + # Semantic actions want store to be at index 2 + withasstmt ::= expr setupwithas store suite_stmts_opt + POP_BLOCK LOAD_CONST COME_FROM with_cleanup + store ::= STORE_NAME + store ::= STORE_FAST # tryelsetmtl doesn't need COME_FROM since the jump might not # be the the join point at the end of the "try" but instead back to the diff --git a/uncompyle6/semantics/customize.py b/uncompyle6/semantics/customize.py index fe3c5688..99b642c4 100644 --- a/uncompyle6/semantics/customize.py +++ b/uncompyle6/semantics/customize.py @@ -71,6 +71,9 @@ def customize_for_version(self, is_pypy, version): TABLE_DIRECT.update( {"except_cond3": ("%|except %c, %c:\n", (1, "expr"), (-2, "store"))} ) + if version <= 2.6: + TABLE_DIRECT["testtrue_then"] = TABLE_DIRECT["testtrue"] + if 2.4 <= version <= 2.6: TABLE_DIRECT.update({"comp_for": (" for %c in %c", 3, 1)}) else: @@ -134,7 +137,6 @@ def customize_for_version(self, is_pypy, version): } ) if version == 2.4: - def n_iftrue_stmt24(node): self.template_engine(("%c", 0), node) self.default(node) From b8f4dca50517158d78d2046d24ff00d235f1f3a6 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 07:19:40 -0500 Subject: [PATCH 03/10] Update README.rsgt --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 63dd2c85..1023400e 100644 --- a/README.rst +++ b/README.rst @@ -223,7 +223,6 @@ There is lots to do, so please dig in and help. See Also -------- -* https://github.com/zrax/pycdc : aims to support all versions of Python, but doesn't currently. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker `_ for details. Currently lightly maintained. * https://github.com/rocky/python-decompile3 : Much smaller and more modern code, focusing on 3.7+. Changes in that will get migrated back ehre. * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained. * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained. @@ -233,6 +232,7 @@ See Also * https://github.com/rocky/python-xdis : Cross Python version disassembler * https://github.com/rocky/python-xasm : Cross Python version assembler * https://github.com/rocky/python-uncompyle6/wiki : Wiki Documents which describe the code and aspects of it in more detail +* https://github.com/zrax/pycdc : The README for this C++ code syas it aims to support all versions of Python. It is best for Python versions around 2.7 and 3.3 when the code was initially developed. Accuracy for current versions of Python3 and early versions of Python is lacking. Without major effort, it is unlikely it can be made to support current Python 3. See its `issue tracker `_ for details. Currently lightly maintained. .. _trepan: https://pypi.python.org/pypi/trepan2g From daab1e8610f1cbe4a22a63f503723556c66c3f9e Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 12:20:00 -0500 Subject: [PATCH 04/10] Start to tolerate 3.9 (in pydisassemble) --- setup.py | 4 +-- uncompyle6/scanner.py | 1 + uncompyle6/scanners/scanner38.py | 9 +++--- uncompyle6/scanners/scanner39.py | 55 ++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 uncompyle6/scanners/scanner39.py diff --git a/setup.py b/setup.py index 4af1f80f..060b692b 100755 --- a/setup.py +++ b/setup.py @@ -4,8 +4,8 @@ import sys """Setup script for the 'uncompyle6' distribution.""" SYS_VERSION = sys.version_info[0:2] -if not ((2, 6) <= SYS_VERSION <= (3, 8)): - mess = "Python Release 2.6 .. 3.8 are supported in this code branch." +if not ((2, 6) <= SYS_VERSION <= (3, 9)): + mess = "Python Release 2.6 .. 3.9 are supported in this code branch." if ((2, 4) <= SYS_VERSION <= (2, 7)): mess += ("\nFor your Python, version %s, use the python-2.4 code/branch." % sys.version[0:3]) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 4a88245d..bf767f9d 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -60,6 +60,7 @@ PYTHON_VERSIONS = frozenset( 3.6, 3.7, 3.8, + 3.9, ) ) diff --git a/uncompyle6/scanners/scanner38.py b/uncompyle6/scanners/scanner38.py index 31512fe7..7477e187 100644 --- a/uncompyle6/scanners/scanner38.py +++ b/uncompyle6/scanners/scanner38.py @@ -12,14 +12,13 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" -Python 3.8 bytecode decompiler scanner +"""Python 3.8 bytecode decompiler scanner -Does some additional massaging of xdis-disassembled instructions to -make things easier for decompilation. +Does some token massaging of xdis-disassembled instructions to make +things easier for decompilation. This sets up opcodes Python's 3.8 and calls a generalized -scanner routine for Python 3. +scanner routine for Python 3.7 and up. """ from uncompyle6.scanners.scanner37 import Scanner37 diff --git a/uncompyle6/scanners/scanner39.py b/uncompyle6/scanners/scanner39.py new file mode 100644 index 00000000..c62bbe3d --- /dev/null +++ b/uncompyle6/scanners/scanner39.py @@ -0,0 +1,55 @@ +# Copyright (c) 2019 by Rocky Bernstein +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +"""Python 3.9 bytecode decompiler scanner. + +Does some token massaging of xdis-disassembled instructions to make +things easier for decompilation. + +This sets up opcodes Python's 3.9 and calls a generalized +scanner routine for Python 3.7 and up. +""" + +from uncompyle6.scanners.scanner38 import Scanner38 +from uncompyle6.scanners.scanner37base import Scanner37Base + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_38 as opc + +# bytecode verification, verify(), uses JUMP_OPS from here +JUMP_OPs = opc.JUMP_OPS + + +class Scanner39(Scanner38): + def __init__(self, show_asm=None): + Scanner37Base.__init__(self, 3.9, show_asm) + return + + pass + + +if __name__ == "__main__": + from uncompyle6 import PYTHON_VERSION + + if PYTHON_VERSION == 3.9: + import inspect + + co = inspect.currentframe().f_code + tokens, customize = Scanner39().ingest(co) + for t in tokens: + print(t.format()) + pass + else: + print("Need to be Python 3.9 to demo; I am %s." % + PYTHON_VERSION) From fdac1e3c46ff7c25b24bfda11496617afce05354 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 12:56:30 -0500 Subject: [PATCH 05/10] Use 3.9-enabled xdis --- __pkginfo__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__pkginfo__.py b/__pkginfo__.py index 8c641ddd..60ec1573 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -58,7 +58,7 @@ entry_points = { ]} ftp_url = None install_requires = ["spark-parser >= 1.8.9, < 1.9.0", - "xdis >= 4.1.3, < 4.2.0"] + "xdis >= 4.2.0, < 4.3.0"] license = "GPL3" mailing_list = "python-debugger@googlegroups.com" From 7d9c4ce8ca02b404386721ada22afef44277c726 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 14:04:57 -0500 Subject: [PATCH 06/10] Better try/else detection --- test/stdlib/runtests.sh | 3 ++- uncompyle6/parsers/parse2.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index acb1cb92..3e74384c 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -105,8 +105,9 @@ case $PYVERSION in [test_generators.py]=1 [test_grp.py]=1 # Long test - might work Control flow? [test_opcodes.py]=1 + [test_pyclbr.py]=1 # Bug in checkModule()/ismethod() try confused for try-else [test_pwd.py]=1 # Long test - might work? Control flow? - [test_re.py]=1 # Probably Control flow? + [test_re.py]=1 # try confused with try-else again [test_queue.py]=1 # Control flow? [test_trace.py]=1 # Line numbers are expected to be different [test_urllib2net.py]=1 # Fails on its own. May need interactive input diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 36a36c06..414f016f 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -639,6 +639,8 @@ class Python2Parser(PythonParser): self.check_reduce["raise_stmt1"] = "tokens" self.check_reduce["assert_expr_and"] = "AST" + self.check_reduce["tryelsestmt"] = "AST" + self.check_reduce["tryelsestmtl"] = "AST" self.check_reduce["aug_assign2"] = "AST" self.check_reduce["or"] = "AST" # self.check_reduce['_stmts'] = 'AST' @@ -674,6 +676,33 @@ class Python2Parser(PythonParser): elif lhs in ("delete_subscript", "del_expr"): op = ast[0][0] return op.kind in ("and", "or") + elif lhs in ("tryelsestmt", "tryelsetmtl"): + # Check the end of the except handler that there isn't a jump from + # inside the except handler to the end. If that happens + # then this is a "try" with no "else". + except_handler = ast[3] + if except_handler == "except_handler": + + come_from = except_handler[-1] + # We only care about the *first* come_from because that is the + # the innermost one. So if the "tryelse" is invalid (should be a "try") + # ti will be invalid here. + if come_from == "COME_FROM": + first_come_from = except_handler[-1] + else: + assert come_from == "come_froms" + first_come_from = come_from[0] + leading_jump = except_handler[0] + + # We really don't care that this is a jump per-se. But + # we could also check that this jumps to the end of the except if + # desired. + if isinstance(leading_jump, SyntaxTree): + except_handler_first_offset = leading_jump.first_child().off2int() + else: + except_handler_first_offset = leading_jump.off2int() + return first_come_from.attr > except_handler_first_offset + return False From 825ed3fef9b77d15ac92a843089dfb459cd1e4ad Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 14:13:30 -0500 Subject: [PATCH 07/10] Reinstate some stdlib tests... now that we distinguish try/else a little better --- test/stdlib/runtests.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 3e74384c..7534b0c3 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -155,7 +155,6 @@ case $PYVERSION in [test_doctest.py]=1 # Fails on its own [test_exceptions.py]=1 [test_format.py]=1 # control flow. uncompyle2 does not have problems here - [test_frozen.py]=1 # try vs try/else control flow. uncompyle2 does not have problems here [test_generators.py]=1 # control flow. uncompyle2 has problem here too [test_grammar.py]=1 # Too many stmts. Handle large stmts [test_grp.py]=1 # test takes to long, works interactively though @@ -169,8 +168,6 @@ case $PYVERSION in [test_modulefinder.py]=1 # FIX [test_multiprocessing.py]=1 # On uncompyle2, takes 24 secs [test_pep352.py]=1 # ? - [test_posix.py]=1 # Bug in try-else detection inside test_initgroups() - # Deal with when we have better flow-control detection [test_pwd.py]=1 # Takes too long [test_pty.py]=1 [test_queue.py]=1 # Control flow? From 6546bbdaf97936b9c8e52ddc17b2e219b21bd6ba Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 16:19:10 -0500 Subject: [PATCH 08/10] Fix 2.x false tryelsestmtl detection... With grammar reduction tests. Same as tryelsestmt. Lots of stdlib tests work now. (More remain though.) --- test/stdlib/runtests.sh | 57 +++++++----------------------------- uncompyle6/parsers/parse2.py | 2 +- 2 files changed, 12 insertions(+), 47 deletions(-) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 7534b0c3..45e74859 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -36,7 +36,7 @@ case $PYVERSION in [test_grp.py]=1 # Long test - might work Control flow? [test_pep247.py]=1 # Long test - might work? Control flow? [test_pwd.py]=1 # Long test - might work? Control flow? - [test_pyclbr.py]=1 # Bug in checkModule()/ismethod() try confused for try-else + [test_pyclbr.py]=1 # Investigate [test_pyexpat.py]=1 # Investigate [test_queue.py]=1 # Control flow? [test_re.py]=1 # try confused with try-else again @@ -55,9 +55,9 @@ case $PYVERSION in [test_grp.py]=1 # Long test - might work Control flow? [test_pdb.py]=1 # Line-number specific [test_pep247.py]=1 # "assert xxx or .." not detected properly in check_hash_module() - [test_pep352.py]=1 # try confused with try-else again; in test_inheritance() + [test_pep352.py]=1 # Investigate [test_pwd.py]=1 # Long test - might work? Control flow? - [test_pyclbr.py]=1 # Bug in checkModule()/ismethod() try confused for try-else + [test_pyclbr.py]=1 # Investigate [test_queue.py]=1 # Control flow? [test_re.py]=1 # Possibly try confused with try-else again [test_struct.py]=1 # "if and" confused for if .. assert and @@ -71,44 +71,16 @@ case $PYVERSION in ;; 2.6) SKIP_TESTS=( - [test_aepack.py]=1 - [test_aifc.py]=1 - [test_array.py]=1 - [test_audioop.py]=1 - [test_base64.py]=1 - [test_bigmem.py]=1 - [test_binascii.py]=1 - [test_builtin.py]=1 - [test_bytes.py]=1 - [test_class.py]=1 - [test_codeccallbacks.py]=1 - [test_codecencodings_cn.py]=1 - [test_codecencodings_hk.py]=1 - [test_codecencodings_jp.py]=1 - [test_codecencodings_kr.py]=1 - [test_codecencodings_tw.py]=1 - [test_codecencodings_cn.py]=1 - [test_codecmaps_hk.py]=1 - [test_codecmaps_jp.py]=1 - [test_codecmaps_kr.py]=1 - [test_codecmaps_tw.py]=1 - [test_codecs.py]=1 + [test_aepack.py]=1 # Fails on its own + [test_codeccallbacks.py]=1 # Fails on its own [test_compile.py]=1 # Intermittent - sometimes works and sometimes doesn't - [test_cookielib.py]=1 - [test_copy.py]=1 - [test_decimal.py]=1 - [test_descr.py]=1 # Problem in pickle.py? [test_exceptions.py]=1 - [test_extcall.py]=1 - [test_float.py]=1 - [test_future4.py]=1 - [test_generators.py]=1 + [test_generators.py]=1 # Investigate [test_grp.py]=1 # Long test - might work Control flow? - [test_opcodes.py]=1 - [test_pyclbr.py]=1 # Bug in checkModule()/ismethod() try confused for try-else + [test_pep352.py]=1 # Investigate + [test_pprint.py]=1 + [test_pyclbr.py]=1 # Investigate [test_pwd.py]=1 # Long test - might work? Control flow? - [test_re.py]=1 # try confused with try-else again - [test_queue.py]=1 # Control flow? [test_trace.py]=1 # Line numbers are expected to be different [test_urllib2net.py]=1 # Fails on its own. May need interactive input [test_zipfile64.py]=1 # Skip Long test @@ -122,8 +94,6 @@ case $PYVERSION in # .pyenv/versions/2.6.9/lib/python2.6/tabnanny.pyc # .pyenv/versions/2.6.9/lib/python2.6/tarfile.pyc - # Not getting set by bach below? - [test_pprint.py]=1 ) if (( batch )) ; then @@ -158,7 +128,7 @@ case $PYVERSION in [test_generators.py]=1 # control flow. uncompyle2 has problem here too [test_grammar.py]=1 # Too many stmts. Handle large stmts [test_grp.py]=1 # test takes to long, works interactively though - [test_hashlib.py]=1 # Investiage + [test_hashlib.py]=1 # Investigate [test_io.py]=1 # Test takes too long to run [test_ioctl.py]=1 # Test takes too long to run [test_long.py]=1 @@ -167,24 +137,19 @@ case $PYVERSION in [test_memoryio.py]=1 # FIX [test_modulefinder.py]=1 # FIX [test_multiprocessing.py]=1 # On uncompyle2, takes 24 secs - [test_pep352.py]=1 # ? [test_pwd.py]=1 # Takes too long [test_pty.py]=1 - [test_queue.py]=1 # Control flow? - [test_re.py]=1 # Probably Control flow? [test_runpy.py]=1 # Long and fails on its own [test_select.py]=1 # Runs okay but takes 11 seconds [test_socket.py]=1 # Runs ok but takes 22 seconds [test_subprocess.py]=1 # Runs ok but takes 22 seconds [test_sys_setprofile.py]=1 [test_sys_settrace.py]=1 # Line numbers are expected to be different - [test_strtod.py]=1 # FIX [test_traceback.py]=1 # Line numbers change - duh. - [test_types.py]=1 # try/else confusions [test_unicode.py]=1 # Too long to run 11 seconds [test_xpickle.py]=1 # Runs ok but takes 72 seconds [test_zipfile64.py]=1 # Runs ok but takes 204 seconds - [test_zipimport.py]=1 # We can't distinguish try from try/else yet + [test_zipimport.py]=1 # FIXME: improper try from try/else ? ) if (( batch )) ; then # Fails in crontab environment? diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 414f016f..7d3331c9 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -676,7 +676,7 @@ class Python2Parser(PythonParser): elif lhs in ("delete_subscript", "del_expr"): op = ast[0][0] return op.kind in ("and", "or") - elif lhs in ("tryelsestmt", "tryelsetmtl"): + elif lhs in ("tryelsestmt", "tryelsestmtl"): # Check the end of the except handler that there isn't a jump from # inside the except handler to the end. If that happens # then this is a "try" with no "else". From edbbefb57d906573a8387c1caa8af144490d2c2e Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 18:28:57 -0500 Subject: [PATCH 09/10] Start to remove crud in control-flow detection for 3.7+ --- uncompyle6/scanners/scanner37base.py | 163 +-------------------------- uncompyle6/semantics/customize38.py | 7 +- 2 files changed, 7 insertions(+), 163 deletions(-) diff --git a/uncompyle6/scanners/scanner37base.py b/uncompyle6/scanners/scanner37base.py index f6fc5a8c..9519724a 100644 --- a/uncompyle6/scanners/scanner37base.py +++ b/uncompyle6/scanners/scanner37base.py @@ -775,184 +775,27 @@ class Scanner37Base(Scanner): } ) elif op in self.pop_jump_tf: - start = offset + inst.inst_size target = inst.argval - rtarget = self.restrict_to_parent(target, parent) prev_op = self.prev_op - # Do not let jump to go out of parent struct bounds - if target != rtarget and parent["type"] == "and/or": - self.fixed_jumps[offset] = rtarget - return - - # Does this jump to right after another conditional jump that is - # not myself? If so, it's part of a larger conditional. - # rocky: if we have a conditional jump to the next instruction, then - # possibly I am "skipping over" a "pass" or null statement. + # FIXME: hack upon hack, test_pysource.py fails with this + # Until the grammar is corrected we do this fiction... pretarget = self.get_inst(prev_op[target]) - if ( pretarget.opcode in self.pop_jump_if_pop and (target > offset) and pretarget.offset != offset ): - # FIXME: hack upon hack... - # In some cases the pretarget can be a jump to the next instruction - # and these aren't and/or's either. We limit to 3.5+ since we experienced there - # but it might be earlier versions, or might be a general principle. if pretarget.argval != target: # FIXME: this is not accurate The commented out below # is what it should be. However grammar rules right now # assume the incorrect offsets. # self.fixed_jumps[offset] = target self.fixed_jumps[offset] = pretarget.offset - self.structs.append( - {"type": "and/or", "start": start, "end": pretarget.offset} - ) return - # The opcode *two* instructions before the target jump offset is important - # in making a determination of what we have. Save that. - pre_rtarget = prev_op[rtarget] - - if op == self.opc.POP_JUMP_IF_FALSE: - self.fixed_jumps[offset] = target - - # op == POP_JUMP_IF_TRUE - else: - next = self.next_stmt[offset] - if prev_op[next] == offset: - pass - elif self.is_jump_forward(next) and target == self.get_target(next): - if code[prev_op[next]] == self.opc.POP_JUMP_IF_FALSE: - if ( - code[next] == self.opc.JUMP_FORWARD - or target != rtarget - or code[prev_op[pre_rtarget]] - not in (self.opc.JUMP_ABSOLUTE, self.opc.RETURN_VALUE) - ): - self.fixed_jumps[offset] = prev_op[next] - return - elif ( - code[next] == self.opc.JUMP_ABSOLUTE - and self.is_jump_forward(target) - and self.get_target(target) == self.get_target(next) - ): - self.fixed_jumps[offset] = prev_op[next] - return - - rtarget_is_ja = code[pre_rtarget] == self.opc.JUMP_ABSOLUTE - if ( - rtarget_is_ja - and pre_rtarget in self.stmts - and pre_rtarget != offset - and prev_op[pre_rtarget] != offset - and not ( - code[rtarget] == self.opc.JUMP_ABSOLUTE - and code[rtarget + 3] == self.opc.POP_BLOCK - and code[prev_op[pre_rtarget]] != self.opc.JUMP_ABSOLUTE - ) - ): - rtarget = pre_rtarget - - # Does the "jump if" jump beyond a jump op? - # That is, we have something like: - # POP_JUMP_IF_FALSE HERE - # ... - # JUMP_FORWARD - # HERE: - # - # If so, this can be block inside an "if" statement - # or a conditional assignment like: - # x = 1 if x else 2 - # - # For 3.5, for JUMP_FORWARD above we could have also - # JUMP_BACK or CONTINUE - # - # There are other situations we may need to consider, like - # if the condition jump is to a forward location. - # Also the existence of a jump to the instruction after "END_FINALLY" - # will distinguish "try/else" from "try". - rtarget_break = (self.opc.RETURN_VALUE, self.opc.BREAK_LOOP) - - if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja): - if_end = self.get_target(pre_rtarget) - - # If the jump target is back, we are looping - if ( - if_end < pre_rtarget - and self.version < 3.8 - and (code[prev_op[if_end]] == self.opc.SETUP_LOOP) - ): - if if_end > start: - return - - end = self.restrict_to_parent(if_end, parent) - - self.structs.append( - {"type": "if-then", "start": start, "end": pre_rtarget} - ) - - # FIXME: add this - # self.fixed_jumps[offset] = rtarget - self.not_continue.add(pre_rtarget) - - if rtarget < end and ( - code[rtarget] not in (self.opc.END_FINALLY, self.opc.JUMP_ABSOLUTE) - and code[prev_op[pre_rtarget]] - not in (self.opc.POP_EXCEPT, self.opc.END_FINALLY) - ): - self.structs.append({"type": "else", "start": rtarget, "end": end}) - self.else_start[rtarget] = end - elif self.is_jump_back(pre_rtarget, 0): - if_end = rtarget - self.structs.append( - {"type": "if-then", "start": start, "end": pre_rtarget} - ) - self.not_continue.add(pre_rtarget) - elif code[pre_rtarget] in rtarget_break: - self.structs.append({"type": "if-then", "start": start, "end": rtarget}) - # It is important to distingish if this return is inside some sort - # except block return - jump_prev = prev_op[offset] - if self.is_pypy and code[jump_prev] == self.opc.COMPARE_OP: - if self.opc.cmp_op[code[jump_prev + 1]] == "exception-match": - return - pass - - # Check that next instruction after pops and jump is - # not from SETUP_EXCEPT - next_op = rtarget - if code[next_op] == self.opc.POP_BLOCK: - next_op += instruction_size(self.code[next_op], self.opc) - if code[next_op] == self.opc.JUMP_ABSOLUTE: - next_op += instruction_size(self.code[next_op], self.opc) - if next_op in targets: - for try_op in targets[next_op]: - come_from_op = code[try_op] - if self.version < 3.8 and come_from_op == self.opc.SETUP_EXCEPT: - return - pass - - self.fixed_jumps[offset] = rtarget - - if code[pre_rtarget] == self.opc.RETURN_VALUE: - # If we are at some sort of POP_JUMP_IF and the instruction before was - # COMPARE_OP exception-match, then pre_rtarget is not an end_if - if not ( - inst_index > 0 - and self.insts[inst_index - 1].argval == "exception-match" - ): - self.return_end_ifs.add(pre_rtarget) - else: - self.fixed_jumps[offset] = rtarget - self.not_continue.add(pre_rtarget) - else: - - if target > offset: - self.fixed_jumps[offset] = target - pass + self.fixed_jumps[offset] = target elif self.version < 3.8 and op == self.opc.SETUP_EXCEPT: target = self.get_target(offset) diff --git a/uncompyle6/semantics/customize38.py b/uncompyle6/semantics/customize38.py index 2e587d93..f21ba083 100644 --- a/uncompyle6/semantics/customize38.py +++ b/uncompyle6/semantics/customize38.py @@ -89,9 +89,10 @@ def customize_for_version38(self, version): (1, "_ifstmts_jumpl") ), 'whilestmt38': ( '%|while %c:\n%+%c%-\n\n', - (1, 'testexpr'), (2, 'l_stmts') ), + (1, 'testexpr'), + 2 ), # "l_stmts" or "pass" 'whileTruestmt38': ( '%|while True:\n%+%c%-\n\n', - (1, 'l_stmts') ), + 1 ), # "l_stmts" or "pass" 'try_elsestmtl38': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-', (1, 'suite_stmts_opt'), @@ -106,7 +107,7 @@ def customize_for_version38(self, version): 'tryfinally38': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', (3, 'returns'), 6 ), - "named_expr": ( # AKA "walrus operatotr" + "named_expr": ( # AKA "walrus operator" "%c := %c", (2, "store"), (0, "expr") ) }) From 3e3dd87c3bfb69f963696c95ba78475d98cab7b3 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 9 Dec 2019 21:37:39 -0500 Subject: [PATCH 10/10] Get ready for release 3.6.0 --- NEWS.md | 21 ++++++++++++++++++++- uncompyle6/version.py | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 67d67107..33e1da34 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,23 @@ -3.5.1 2019-10-29 JNC +3.6.0: 2019-12-10 gecko gecko +============================= + +The main focus in this release was more accurate decompilation especially +for 3.7 and 3.8. However there are some improvments to Python 2.x as well, +including one of the long-standing problems of detecting the difference between +`try ... ` and `try else ...`. + +With this release we now rebase Python 3.7 on off of a 3.7 base; This +is also as it is (now) in decompyle3. This facilitates removing some of the +cruft in control-flow detection in the 2.7 uncompyle2 base. + +Alas, decompilation speed for 3.7 on is greatly increased. Hopefull +this is temporary (cough, cough) until we can do a static control flow +pass. + +Finally, runing in 3.9-dev is tolerated. We can disassemble, but no parse tables yet. + + +3.5.1 2019-11-17 JNC ==================== - Pypy 3.3, 3.5, 3.6, and 3.6.9 support diff --git a/uncompyle6/version.py b/uncompyle6/version.py index bbb23736..7e7de6f6 100644 --- a/uncompyle6/version.py +++ b/uncompyle6/version.py @@ -12,4 +12,4 @@ # along with this program. If not, see . # This file is suitable for sourcing inside bash as # well as importing into Python -VERSION="3.5.1" # noqa +VERSION="3.6.0" # noqa