From 9746b21bbf1c69c01c53a562a8c2c22e2e49d037 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 17 Apr 2023 23:20:42 -0400 Subject: [PATCH 01/11] Update 2.7 literal test --- test/bytecode_2.7_run/05_long_literals.pyc | Bin 20615 -> 20722 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/test/bytecode_2.7_run/05_long_literals.pyc b/test/bytecode_2.7_run/05_long_literals.pyc index d5d998e6eb4b06307ca8b7a51e186e100a407287..21d4540e5d957c1630dc03d943e2a10a95ea0ec1 100644 GIT binary patch delta 3784 zcmXZfWtdd;8;9|ko#&w&c8T4kOF){HmIeWdrE`hzN=hz>gd-&a2Dl0eNP|T$EvlFQ~=VL3PM^_VMu2x3h7P7A%m$z7&DqmK_*ja$ZRSLSxn_2tEnPn zGsQu6Q)S3ustS>&>JVkB2{}!*AeX5Qv;#I_9qU7?t%8x%M7fM-mC5~i1+q^UQQGWCULP5q&? zX#kWl4T7?!A%VEC%9)ZF%bSKn1=9$qXc`HXOrs&r^cp;8dL1g8#z7U+1gL6y1FD(c zgzBa#P{Z^V)HJ;V&zq(}Ez^5Y+w?xvG0lK*UDJnH&om3_o8~|R(_Cn1nh%Xk3*iOR zVrXny3QbI(LQ~V{@Sw-lj{?$8;I`ntp+PreC4I>34Y9bQK1euERjnpD@UD3kIA1f+426Fx2!n zB$*z-Fw-L#Zu$paF$IEg8exinS50BVk)~8I%9I91o6^A;QwDg=lnJ;_S>Sb3HW+Kl z0pm&Xx{kRfa&L)b)yu$c_uOEQElWC&Zy5VnyaY$rq5L58rC z3?aOW4)H58gs;gEc9S9OAw$?phVTs;!nb4y-;p7FPlm9M3}HVR!VhE!2gnc(k|7)- zLpV%^aD)uuD9_bqJVu6aoDAUv8Nx|2gi~Y)r^yh`kRkj?hH#b);T##lc`}3xWC$0@ z5H67+{6vOunGE4)GK62~?;Kt+U4UQ75Pl;=_?-;l4>E+SWC+*D5U!IU+#o~vlMLY| z8Nw|xgxh2Ye~}^FAw#%JhH#GzBjWTo{YBt?lfeTrgok7ZkH`=nlOg;=hVX<8A>hge zL01lla797Ll?%+32iz3{sa$y>wJSfQaTSELuELPcRWyw0UBw}Ts{~|pm4ZyJ(vaCz z7P7d?LsnNs$mWWJ?5@g?!&Ma`UDYAVRTFZ$YC$en9mwsf2YFl#AllUkVqA?Q=r7jQ zlrgWX8RT=dfc&m_DBx-h1zl~SkgGiuc6EdzuFg=@)fI}lxbqt`1J}pU z&@~SlxfZ|+u0_z;wFH{DK7ppL&)`MZ7tqYL9GbgULJQYwXz5CUc-J~;6?SdF)~-#^ z#`Pt%b!~-quIjcPk3KBi6Gx(D0Ec9}nhu*G>(8u)?^mY9V{aja|zw0-6+4Tnua9x9et{X7ObrS}= zZo?4Q9T@7m2T88`FwFH3hPxiaE8M~-IKmYS@j7&cV5G}olq)ricBO?euJrJlD0QlR{@ypDip>kt|BniRSe#8Jp*sM zO2Rv?XW?B}8JOlO2h&{@;5}C*NOnC3@4KqN2d-)`!&L)jx}Jv*UA5sOS6!Ipst>bW z4MX%d$Mpi^$F3$Y*YzUIb2W$gu9mRC)e086+Q1@LJ6P=M083n*V5zGMeBw%gPstEI zBSZL{4B-nhgk@w1%gHcAe=F#aaU~hTDl&xCWC&}>5K_nx){-HtBSTnEhOmKq2ph=| zHjyE0CPVm=3}Fiy!d5bbZDa`B$q>Rj=n!|3A?zYU_=*hSYcho0WC(l65cZNGd_#ut zEg8ahWC-7rA?zbV*iVM=0~x{rGK7O<2#3fJ4)Y|f!y{w}N68S5ks%x>LpVW(aFPt+ z6dA&4GK4c^2tSe`oFzjzM}}~o4B-M9!bLKKOJoQ?ks)06?mC4(yUxHbWC&Nt5Pl^? z_>BzVcQS-O$PliQAzULvxK4&}gACzMGK8CC2)B~^L`Mg2Cr^wn7W^yuVs!T89ntxN zcTz4!#|0bQ59vWj4?~Gr0(unc<$;cA0*Pq?iU{Q9#M6yoq1gX#44N9Y_ziT7@J&sK2RqIyT$gG@BO3fi5 pK?8dxB*X_I0)ga#ji-716aFCiN#pon<&^j)l{{5Bj|@fy{|EY`H?sf$ delta 3674 zcmXZfWtde}8;9|8<~$GG4Ks9i58X(MbO{Xcr==0;LyCZe4oV85G#D_1gwlwB(k=FzpO(0q?LrM#;LP`g(LrM>CLdpnlL&^m2LW+cdDKi93 zSs=oc4ML{uV5S`4rd$xuln3IQ@<9SqfiNaC6@o;jB9Pcr43d~iKvGjFNM?$LM7Kvq*L$YyE-*-dRBhp9c}GEN|;7LNz-U3Wf}{mP2(Zj^eL1veFkMslcAjH zb0}|`1{F**prYwZsAQT6l}%qmxQb~uRyECqYNq*6-LwE|n7)OYrtjcA(_*M)S_-vI z%b|{GCDb*ohI*zSp}y&7Xkc0k@0->^L(>NM!1N0=GHr&&VbfM@V)_l5nzlnT(@tn^ z+6^sCd!eOiAG9*gMOx$(BJeL z2AJN$KvN(XtwE*;_}COS9BhgQpO_NB5K|%;YDxmbOvzxlDFuu$rGk;BG%(7P4)})| zV6-U{j4?&QSZaiE)Cl9r5GIfzd`gBekqpE586CnTGK9%w2vf)qJ|{z%N`^3v3}HGM z!VEHmFUSzSBtwWLLzqd1@D&-t*JKE@$Pi|eALpVW(aFPt+6dA&4GK4c^2xrL<&XFPfO@?ru4B-M9!bLKK zOJoR_$q=s4-vzvCx&+tA5U!IU+#o}^NrrHX4B<8z!W}Y%yJQIW$Pn(6Av_>Mcu0ou zhz#K$GK9xu2v5i`B3e)BF9QEH89XCHcut1!9~r_6GK8082(QQxUXvlbAwzgehVYII zA>hgcL01$+xUxXVl?}|51KgDh;<@sKF}^DwBybgggswu6$W;UqyNW>)R|!b!Dh0`0 z(U9C#7E-v%LrPagNad;wsa;hejjKAOb=8D)u3C`ZRR=P->P66BMppyIOs<9y>1qT~ zt|pM#)eN$@T0mA;E6C<*1KC||A&09yH+y(y`X@r zcLetiVD`UHx(hC*@Ia46v#2_>Dy<1}=QfzqyV5bc@(Wn2@X ztZNdKb4`KruBlMLH61FtzJPEgS1eX`eFarqv!JSL4pei^gX*qtpoVK9)O0O^_gvpY zE!PsL?OFzPTq~fiYZcUU{Q&h{KS2Z68hGCo2Mt~8;e)ViBQ|nvg2t{b(8TpCG<9u* zX09F3+_ei@xb{Fx*YD8E^#`w@*x(r=hSD~xxI&^d0gzm1}(8F~Xdb;jIFV{o($n_8Oc0GYUu79Dg>pApu zy@39%S1`c!1_rW(?{JVS7~(i|g@*E(UliIbLEFgu7WVxRT!qYio)lv z;xN@!5~jII!*o{}nBgi1U$`p3m##_>>#71XUDe<#R}J{u^&ZS})eh0$Y*$^zIj;IJ z*Y!TkbA15wU5()zS5sKvY7PrsE#X^NYgpv^5WaJ@gYR82u$T;C2^qptGK6Jh2+PS3 zR*+$c{#Mc<<0>+Q)no`ikRkj?hVT;^!p~#~Yse7Rk|D&ghOmwdVLchb1~P<=WC*{I zA#5T;*i447g$yCQl@9S&GKAmA5VnyaY$rq5L58rC3}F`;!frBzJ!A-b$q;@gL)b@# z@CO;felmmuWC#bz5dI`X_>0}L9uJWr9413JLWXdZ4B;3V!f`T$6J!V{$q-JFA)F>d zI75bTmJHz>8N%OW2f1RcW)PRfz>wHO^=8DXev@E{xQ+Eo`~Ltudm|$N From 22373b4195d69d643cf40c3c1bf29f112762d6d7 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 17 Apr 2023 23:35:00 -0400 Subject: [PATCH 02/11] Update 2.5 stdlib excludes --- test/stdlib/2.5-exclude.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/stdlib/2.5-exclude.sh b/test/stdlib/2.5-exclude.sh index 1b0d1afa..5dc11c56 100644 --- a/test/stdlib/2.5-exclude.sh +++ b/test/stdlib/2.5-exclude.sh @@ -25,7 +25,6 @@ SKIP_TESTS=( [test_nis.py]=1 # it fails on its own [test_normalization.py]=1 # it fails on its own [test_ossaudiodev.py]=1 # it fails on its own - [test_pep277.py]=1 # it fails on its own [test_plistlib.py]=1 # it fails on its own [test_rgbimg.py]=1 # it fails on its own [test_scriptpackages.py]=1 # it fails on its own From 54776275c06028df2f4e5f2ab4e659d1349b52c3 Mon Sep 17 00:00:00 2001 From: Andre Eberle Date: Tue, 18 Apr 2023 00:12:00 -0400 Subject: [PATCH 03/11] Modified n_actions.py to issue __repr__ on py2 and __str__ py3, should fix the extra quotes --- uncompyle6/semantics/n_actions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncompyle6/semantics/n_actions.py b/uncompyle6/semantics/n_actions.py index 78132c79..54a160ba 100644 --- a/uncompyle6/semantics/n_actions.py +++ b/uncompyle6/semantics/n_actions.py @@ -267,7 +267,10 @@ class NonterminalActions: if elem == "add_value": elem = elem[0] if elem == "ADD_VALUE": - value = "%r" % elem.pattr + if self.version[0] == 2: + value = "%r" % elem.pattr + else: + value = "%s" % elem.pattr else: assert elem.kind == "ADD_VALUE_VAR" value = "%s" % elem.pattr From c01ab5e0012a188d2dfaf4223a8766ab61fe2d3f Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 19 Apr 2023 02:08:36 -0400 Subject: [PATCH 04/11] Tweaks to long-literal handling... * Use version tuple comparison for version testing * small lintin of n_actions * revise test so assert is not removed in 3.8 --- test/bytecode_3.8_run/05_long_literals.pyc | Bin 14651 -> 14801 bytes .../expression/05_long_literals.py | 7 +++++-- uncompyle6/semantics/n_actions.py | 11 ++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/bytecode_3.8_run/05_long_literals.pyc b/test/bytecode_3.8_run/05_long_literals.pyc index 665b3767856cd8bb121c49b6f6d009719174abec..38c0ed30f9ca9524f70b59e312d43a1af8c9665d 100644 GIT binary patch delta 293 zcmdm8bg`H>l$V!_0SKP9*r$A+$Scb@XQOr^pF;~n6zc@06b2w>1Y)LCHXzBI&Kkv- z%9_fU!XOEjX=Y+%0P~q78B*Cmya|jczk?YxST-->doOF2%mgwT3P5Z|Aa(}gViq8g z!Vt`$$>_I|L6hkgW5q4T$|7x`e3AC#%Q{AuJc-4{sYNC6sfDG9IX8iNi@|0w7U_f3 z=mLpboURqgsRbpO`FTY;AU=-=XG))Zp`0B*Y{<^TWy diff --git a/test/simple_source/expression/05_long_literals.py b/test/simple_source/expression/05_long_literals.py index 24c4e3c6..f03d4493 100644 --- a/test/simple_source/expression/05_long_literals.py +++ b/test/simple_source/expression/05_long_literals.py @@ -726,9 +726,12 @@ values = { assert sorted(values.values())[1:] == list(range(2, 34)) +def assert_equal(x, y): + assert x == y # Check that we can distinguish names from strings in literal collections, e.g. lists. # The list has to have more than 4 items to get accumulated in a collection a = ["y", 'Exception', "x", Exception, "z"] -assert a[1] == "Exception" -assert a[3] == Exception + +assert_equal(a[1], "Exception") +assert_equal(a[3], Exception) diff --git a/uncompyle6/semantics/n_actions.py b/uncompyle6/semantics/n_actions.py index 54a160ba..44cf12c7 100644 --- a/uncompyle6/semantics/n_actions.py +++ b/uncompyle6/semantics/n_actions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 by Rocky Bernstein +# Copyright (c) 2022-2023 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -159,7 +159,7 @@ class NonterminalActions: # * class_name - the name of the class # * subclass_info - the parameters to the class e.g. # class Foo(bar, baz) - # ----------- + # ------------ # * subclass_code - the code for the subclass body if node == "classdefdeco2": @@ -181,7 +181,7 @@ class NonterminalActions: subclass_code = build_class[-3][1].attr class_name = node[0][0].pattr else: - raise "Internal Error n_classdef: cannot find class name" + raise RuntimeError("Internal Error n_classdef: cannot find class name") if node == "classdefdeco2": self.write("\n") @@ -228,7 +228,8 @@ class NonterminalActions: else: # from trepan.api import debug; debug() raise TypeError( - f"Internal Error: n_const_list expects dict, list set, or set; got {lastnodetype}" + ("Internal Error: n_const_list expects dict, list set, or set; got " + f"{lastnodetype}") ) self.indent_more(INDENT_PER_LEVEL) @@ -267,7 +268,7 @@ class NonterminalActions: if elem == "add_value": elem = elem[0] if elem == "ADD_VALUE": - if self.version[0] == 2: + if self.version < (3, 0, 0): value = "%r" % elem.pattr else: value = "%s" % elem.pattr From 286bb5948c799e61e4101a47297c0343f8b81ccc Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 30 Apr 2023 22:19:27 -0400 Subject: [PATCH 05/11] Go over bug-report template --- .github/ISSUE_TEMPLATE/bug-report.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 1ded57ca..c1aaec72 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -7,6 +7,11 @@ about: Tell us about uncompyle6 bugs + ## Additional Context From ebcc12e2c329f170c8503500c5da663c72631b63 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 29 May 2023 10:36:16 -0400 Subject: [PATCH 06/11] Misc lint things --- uncompyle6/parser.py | 32 +++++++++++++++++--------------- uncompyle6/parsers/parse24.py | 6 +++--- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index e694f944..97dc5f9d 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2022 Rocky Bernstein +# Copyright (c) 2015-2023 Rocky Bernstein # Copyright (c) 2005 by Dan Pascu # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock @@ -44,8 +44,8 @@ def nop_func(self, args): class PythonParser(GenericASTBuilder): - def __init__(self, SyntaxTree, start, debug): - super(PythonParser, self).__init__(SyntaxTree, start, debug) + def __init__(self, syntax_tree_class, start, debug): + super(PythonParser, self).__init__(syntax_tree_class, start, debug) # FIXME: customize per python parser version # These are the non-terminals we should collect into a list. @@ -103,6 +103,7 @@ class PythonParser(GenericASTBuilder): ) # Instructions filled in from scanner self.insts = [] + self.version = tuple() def ast_first_offset(self, ast): if hasattr(ast, "offset"): @@ -151,9 +152,9 @@ class PythonParser(GenericASTBuilder): Remove recursive references to allow garbage collector to collect this object. """ - for dict in (self.rule2func, self.rules, self.rule2name): - for i in list(dict.keys()): - dict[i] = None + for rule_dict in (self.rule2func, self.rules, self.rule2name): + for i in list(rule_dict.keys()): + rule_dict[i] = None for i in dir(self): setattr(self, i, None) @@ -164,11 +165,11 @@ class PythonParser(GenericASTBuilder): def fix(c): s = str(c) - last_token_pos = s.find("_") - if last_token_pos == -1: + token_pos = s.find("_") + if token_pos == -1: return s else: - return s[:last_token_pos] + return s[:token_pos] prefix = "" if parent and tokens: @@ -267,13 +268,13 @@ class PythonParser(GenericASTBuilder): print(children) return GenericASTBuilder.ambiguity(self, children) - def resolve(self, list): - if len(list) == 2 and "function_def" in list and "assign" in list: + def resolve(self, rule: list): + if len(rule) == 2 and "function_def" in rule and "assign" in rule: return "function_def" - if "grammar" in list and "expr" in list: + if "grammar" in rule and "expr" in rule: return "expr" - # print >> sys.stderr, 'resolve', str(list) - return GenericASTBuilder.resolve(self, list) + # print >> sys.stderr, 'resolve', str(rule) + return GenericASTBuilder.resolve(self, rule) ############################################### # Common Python 2 and Python 3 grammar rules # @@ -667,7 +668,7 @@ def get_python_parser( if compile_mode == "exec": p = parse10.Python10Parser(debug_parser) else: - p = parse10.Python01ParserSingle(debug_parser) + p = parse10.Python10ParserSingle(debug_parser) elif version == (1, 1): import uncompyle6.parsers.parse11 as parse11 @@ -873,6 +874,7 @@ def python_parser( :param showasm: Flag which determines whether the disassembled and ingested code is written to sys.stdout or not. :param parser_debug: dict containing debug flags for the spark parser. + :param is_pypy: True if we are running PyPY :return: Abstract syntax tree representation of the code object. """ diff --git a/uncompyle6/parsers/parse24.py b/uncompyle6/parsers/parse24.py index 31c00296..1958ceb7 100644 --- a/uncompyle6/parsers/parse24.py +++ b/uncompyle6/parsers/parse24.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2018, 2020, 2022 Rocky Bernstein +# Copyright (c) 2016-2018, 2020, 2022-2023 Rocky Bernstein """ spark grammar differences over Python2.5 for Python 2.4. """ @@ -115,8 +115,8 @@ class Python24Parser(Python25Parser): lhs = rule[0] if lhs == "nop_stmt": - l = len(tokens) - if 0 <= l < len(tokens): + token_len = len(tokens) + if 0 <= token_len < len(tokens): return not int(tokens[first].pattr) == tokens[last].offset elif lhs == "try_except": if last == len(tokens): From 41d26bde79c80da50fb7d080f7123ba626be8285 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 29 May 2023 11:00:44 -0400 Subject: [PATCH 07/11] Lint some files --- uncompyle6/bin/pydisassemble.py | 38 ++++--- uncompyle6/bin/uncompile.py | 193 +++++++++++++++++++------------- uncompyle6/main.py | 31 ++--- uncompyle6/parsers/parse13.py | 10 +- uncompyle6/verify.py | 2 +- 5 files changed, 162 insertions(+), 112 deletions(-) diff --git a/uncompyle6/bin/pydisassemble.py b/uncompyle6/bin/pydisassemble.py index 219158fc..a2ea3659 100755 --- a/uncompyle6/bin/pydisassemble.py +++ b/uncompyle6/bin/pydisassemble.py @@ -1,10 +1,13 @@ #!/usr/bin/env python # Mode: -*- python -*- # -# Copyright (c) 2015-2016, 2018, 2020, 2022 by Rocky Bernstein +# Copyright (c) 2015-2016, 2018, 2020, 2022-2023 by Rocky Bernstein # from __future__ import print_function -import sys, os, getopt + +import getopt +import os +import sys from uncompyle6.code_fns import disassemble_file from uncompyle6.version import __version__ @@ -40,13 +43,19 @@ Options: -V | --version show version and stop -h | --help show this message -""".format(program) +""".format( + program +) + +PATTERNS = ("*.pyc", "*.pyo") -PATTERNS = ('*.pyc', '*.pyo') def main(): - Usage_short = """usage: %s FILE... -Type -h for for full help.""" % program + Usage_short = ( + """usage: %s FILE... +Type -h for for full help.""" + % program + ) if len(sys.argv) == 1: print("No file(s) given", file=sys.stderr) @@ -54,17 +63,18 @@ Type -h for for full help.""" % program sys.exit(1) try: - opts, files = getopt.getopt(sys.argv[1:], 'hVU', - ['help', 'version', 'uncompyle6']) + opts, files = getopt.getopt( + sys.argv[1:], "hVU", ["help", "version", "uncompyle6"] + ) except getopt.GetoptError as e: - print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) + print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) for opt, val in opts: - if opt in ('-h', '--help'): + if opt in ("-h", "--help"): print(__doc__) sys.exit(1) - elif opt in ('-V', '--version'): + elif opt in ("-V", "--version"): print("%s %s" % (program, __version__)) sys.exit(0) else: @@ -76,11 +86,11 @@ Type -h for for full help.""" % program if os.path.exists(files[0]): disassemble_file(file, sys.stdout) else: - print("Can't read %s - skipping" % files[0], - file=sys.stderr) + print("Can't read %s - skipping" % files[0], file=sys.stderr) pass pass return -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index fd3242d5..d5beaaad 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -1,14 +1,19 @@ #!/usr/bin/env python # Mode: -*- python -*- # -# Copyright (c) 2015-2017, 2019-2020 by Rocky Bernstein +# Copyright (c) 2015-2017, 2019-2020, 2023 by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # from __future__ import print_function -import sys, os, getopt, time + +import getopt +import os +import sys +import time + from xdis.version_info import version_tuple_to_str -program = 'uncompyle6' +program = "uncompyle6" __doc__ = """ Usage: @@ -58,101 +63,120 @@ Extensions of generated files: '.pyc_dis' '.pyo_dis' successfully decompiled (and verified if --verify) + '_unverified' successfully decompile but --verify failed + '_failed' decompile failed (contact author for enhancement) -""" % ((program,) * 5) +""" % ( + (program,) * 5 +) -program = 'uncompyle6' +program = "uncompyle6" from uncompyle6 import verify from uncompyle6.main import main, status_msg from uncompyle6.version import __version__ + def usage(): print(__doc__) sys.exit(1) def main_bin(): - if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0), - (3, 1), (3, 2), (3, 3), - (3, 4), (3, 5), (3, 6), - (3, 7), (3, 8), (3, 9), (3, 10) - )): + if not ( + sys.version_info[0:2] + in ( + (2, 6), + (2, 7), + (3, 0), + (3, 1), + (3, 2), + (3, 3), + (3, 4), + (3, 5), + (3, 6), + (3, 7), + (3, 8), + (3, 9), + (3, 10), + ) + ): print( f"Error: {program} can decompile only bytecode from Python 3.7" f""" to 3.8.\n\tYou have version: {version_tuple_to_str()}.""" ) sys.exit(-1) - do_verify = recurse_dirs = False + recurse_dirs = False numproc = 0 - outfile = '-' + outfile = "-" out_base = None source_paths = [] timestamp = False timestampfmt = "# %Y.%m.%d %H:%M:%S %Z" try: - opts, pyc_paths = getopt.getopt(sys.argv[1:], 'hac:gtTdrVo:p:', - 'help asm compile= grammar linemaps recurse ' - 'timestamp tree= tree+ ' - 'fragments verify verify-run version ' - 'syntax-verify ' - 'showgrammar encoding='.split(' ')) + opts, pyc_paths = getopt.getopt( + sys.argv[1:], + "hac:gtTdrVo:p:", + "help asm compile= grammar linemaps recurse " + "timestamp tree= tree+ " + "fragments verify verify-run version " + "syntax-verify " + "showgrammar encoding=".split(" "), + ) except getopt.GetoptError as e: - print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr) + print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr) sys.exit(-1) options = {} for opt, val in opts: - if opt in ('-h', '--help'): + if opt in ("-h", "--help"): print(__doc__) sys.exit(0) - elif opt in ('-V', '--version'): + elif opt in ("-V", "--version"): print("%s %s" % (program, __version__)) sys.exit(0) - elif opt == '--verify': - options['do_verify'] = 'strong' - elif opt == '--syntax-verify': - options['do_verify'] = 'weak' - elif opt == '--fragments': - options['do_fragments'] = True - elif opt == '--verify-run': - options['do_verify'] = 'verify-run' - elif opt == '--linemaps': - options['do_linemaps'] = True - elif opt in ('--asm', '-a'): - options['showasm'] = 'after' - options['do_verify'] = None - elif opt in ('--tree', '-t'): - if 'showast' not in options: - options['showast'] = {} - if val == 'before': - options['showast'][val] = True - elif val == 'after': - options['showast'][val] = True + elif opt == "--verify": + options["do_verify"] = "strong" + elif opt == "--syntax-verify": + options["do_verify"] = "weak" + elif opt == "--fragments": + options["do_fragments"] = True + elif opt == "--verify-run": + options["do_verify"] = "verify-run" + elif opt == "--linemaps": + options["do_linemaps"] = True + elif opt in ("--asm", "-a"): + options["showasm"] = "after" + options["do_verify"] = None + elif opt in ("--tree", "-t"): + if "showast" not in options: + options["showast"] = {} + if val == "before": + options["showast"][val] = True + elif val == "after": + options["showast"][val] = True else: - options['showast']['before'] = True - options['do_verify'] = None - elif opt in ('--tree+', '-T'): - if 'showast' not in options: - options['showast'] = {} - options['showast']['after'] = True - options['showast']['before'] = True - options['do_verify'] = None - elif opt in ('--grammar', '-g'): - options['showgrammar'] = True - elif opt == '-o': + options["showast"]["before"] = True + options["do_verify"] = None + elif opt in ("--tree+", "-T"): + if "showast" not in options: + options["showast"] = {} + options["showast"]["after"] = True + options["showast"]["before"] = True + options["do_verify"] = None + elif opt in ("--grammar", "-g"): + options["showgrammar"] = True + elif opt == "-o": outfile = val - elif opt in ('--timestamp', '-d'): + elif opt in ("--timestamp", "-d"): timestamp = True - elif opt in ('--compile', '-c'): + elif opt in ("--compile", "-c"): source_paths.append(val) - elif opt == '-p': + elif opt == "-p": numproc = int(val) - elif opt in ('--recurse', '-r'): + elif opt in ("--recurse", "-r"): recurse_dirs = True - elif opt == '--encoding': - options['source_encoding'] = val + elif opt == "--encoding": + options["source_encoding"] = val else: print(opt, file=sys.stderr) usage() @@ -164,7 +188,7 @@ def main_bin(): if os.path.isdir(f): for root, _, dir_files in os.walk(f): for df in dir_files: - if df.endswith('.pyc') or df.endswith('.pyo'): + if df.endswith(".pyc") or df.endswith(".pyo"): expanded_files.append(os.path.join(root, df)) pyc_paths = expanded_files @@ -175,36 +199,39 @@ def main_bin(): if src_base[-1:] != os.sep: src_base = os.path.dirname(src_base) if src_base: - sb_len = len( os.path.join(src_base, '') ) + sb_len = len(os.path.join(src_base, "")) pyc_paths = [f[sb_len:] for f in pyc_paths] if not pyc_paths and not source_paths: print("No input files given to decompile", file=sys.stderr) usage() - if outfile == '-': - outfile = None # use stdout + if outfile == "-": + outfile = None # use stdout elif outfile and os.path.isdir(outfile): - out_base = outfile; outfile = None + out_base = outfile + outfile = None elif outfile and len(pyc_paths) > 1: - out_base = outfile; outfile = None + out_base = outfile + outfile = None if timestamp: print(time.strftime(timestampfmt)) if numproc <= 1: try: - result = main(src_base, out_base, pyc_paths, source_paths, outfile, - **options) - result = [options.get('do_verify', None)] + list(result) + result = main( + src_base, out_base, pyc_paths, source_paths, outfile, **options + ) + result = [options.get("do_verify", None)] + list(result) if len(pyc_paths) > 1: mess = status_msg(*result) - print('# ' + mess) + print("# " + mess) pass except ImportError as e: print(str(e)) sys.exit(2) - except (KeyboardInterrupt): + except KeyboardInterrupt: pass except verify.VerifyCmpError: raise @@ -216,7 +243,7 @@ def main_bin(): except ImportError: from queue import Empty - fqueue = Queue(len(pyc_paths)+numproc) + fqueue = Queue(len(pyc_paths) + numproc) for f in pyc_paths: fqueue.put(f) for i in range(numproc): @@ -226,13 +253,17 @@ def main_bin(): def process_func(): try: - (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) + (tot_files, okay_files, failed_files, verify_failed_files) = ( + 0, + 0, + 0, + 0, + ) while 1: f = fqueue.get() if f is None: break - (t, o, f, v) = \ - main(src_base, out_base, [f], [], outfile, **options) + (t, o, f, v) = main(src_base, out_base, [f], [], outfile, **options) tot_files += t okay_files += o failed_files += f @@ -249,7 +280,12 @@ def main_bin(): for p in procs: p.join() try: - (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0) + (tot_files, okay_files, failed_files, verify_failed_files) = ( + 0, + 0, + 0, + 0, + ) while True: (t, o, f, v) = rqueue.get(False) tot_files += t @@ -258,8 +294,10 @@ def main_bin(): verify_failed_files += v except Empty: pass - print('# decompiled %i files: %i okay, %i failed, %i verify failed' % - (tot_files, okay_files, failed_files, verify_failed_files)) + print( + "# decompiled %i files: %i okay, %i failed, %i verify failed" + % (tot_files, okay_files, failed_files, verify_failed_files) + ) except (KeyboardInterrupt, OSError): pass @@ -268,5 +306,6 @@ def main_bin(): return -if __name__ == '__main__': + +if __name__ == "__main__": main_bin() diff --git a/uncompyle6/main.py b/uncompyle6/main.py index edf29096..72b86010 100644 --- a/uncompyle6/main.py +++ b/uncompyle6/main.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018-2022 Rocky Bernstein +# Copyright (C) 2018-2023 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,24 +13,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import datetime +import os +import py_compile +import sys from typing import Any, Tuple -import datetime, py_compile, os, sys from xdis import iscode +from xdis.load import load_module from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str + from uncompyle6.code_fns import check_object_path -from uncompyle6.semantics import pysource -from uncompyle6.semantics.pysource import PARSER_DEFAULT_DEBUG from uncompyle6.parser import ParserError +from uncompyle6.semantics import pysource +from uncompyle6.semantics.fragments import code_deparse as code_deparse_fragments +from uncompyle6.semantics.linemap import deparse_code_with_map +from uncompyle6.semantics.pysource import PARSER_DEFAULT_DEBUG, code_deparse from uncompyle6.version import __version__ # from uncompyle6.linenumbers import line_number_mapping -from uncompyle6.semantics.pysource import code_deparse -from uncompyle6.semantics.fragments import code_deparse as code_deparse_fragments -from uncompyle6.semantics.linemap import deparse_code_with_map - -from xdis.load import load_module def _get_outstream(outfile: str) -> Any: dir = os.path.dirname(outfile) @@ -55,7 +57,7 @@ def decompile( source_encoding=None, code_objects={}, source_size=None, - is_pypy=None, + is_pypy=False, magic_int=None, mapstream=None, do_fragments=False, @@ -145,8 +147,8 @@ def decompile( out, bytecode_version, debug_opts=debug_opts, - is_pypy=is_pypy, compile_mode=compile_mode, + is_pypy=is_pypy, ) pass return deparsed @@ -188,7 +190,7 @@ def decompile_file( filename = check_object_path(filename) code_objects = {} - (version, timestamp, magic_int, co, is_pypy, source_size, sip_hash) = load_module( + version, timestamp, magic_int, co, is_pypy, source_size, _ = load_module( filename, code_objects ) @@ -394,14 +396,14 @@ def main( try: # FIXME: Something is weird with Pypy here sys.stdout.flush() - except: + except Exception: pass if current_outfile: sys.stdout.write("\n") try: # FIXME: Something is weird with Pypy here sys.stdout.flush() - except: + except Exception: pass pass return (tot_files, okay_files, failed_files, verify_failed_files) @@ -417,7 +419,6 @@ if sys.platform.startswith("linux") and os.uname()[2][:2] in ["2.", "3.", "4."]: mi.close() return int(mu) / 1000000 - else: def __memUsage(): diff --git a/uncompyle6/parsers/parse13.py b/uncompyle6/parsers/parse13.py index 593805ee..307af36b 100644 --- a/uncompyle6/parsers/parse13.py +++ b/uncompyle6/parsers/parse13.py @@ -1,11 +1,12 @@ -# Copyright (c) 2018 Rocky Bernstein +# Copyright (c) 2018, 2023 Rocky Bernstein from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG + from uncompyle6.parser import PythonParserSingle from uncompyle6.parsers.parse14 import Python14Parser -class Python13Parser(Python14Parser): +class Python13Parser(Python14Parser): def p_misc13(self, args): """ # Nothing here yet, but will need to add LOAD_GLOBALS @@ -24,7 +25,6 @@ class Python13Parser(Python14Parser): # """) # self.check_reduce['doc_junk'] = 'tokens' - # def reduce_is_invalid(self, rule, ast, tokens, first, last): # invalid = super(Python14Parser, # self).reduce_is_invalid(rule, ast, @@ -35,11 +35,11 @@ class Python13Parser(Python14Parser): # return not isinstance(tokens[first].pattr, str) - class Python13ParserSingle(Python13Parser, PythonParserSingle): pass -if __name__ == '__main__': + +if __name__ == "__main__": # Check grammar p = Python13Parser() p.check_grammar() diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index 285b2dae..c30ccf1d 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -411,7 +411,7 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, verify, name=""): check_jumps[dest1].append((i1, i2, dest2)) else: check_jumps[dest1] = [(i1, i2, dest2)] - except: + except Exception: pass i1 += 1 From b0086460de3a6da0430a8d7cdc24de2f8e78cb91 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 15 Jun 2023 21:32:17 -0400 Subject: [PATCH 08/11] Exit when version is not supported --- admin-tools/pyenv-newest-versions | 2 +- uncompyle6/scanners/tok.py | 1 + uncompyle6/verify.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/admin-tools/pyenv-newest-versions b/admin-tools/pyenv-newest-versions index f5911863..f8a24d31 100644 --- a/admin-tools/pyenv-newest-versions +++ b/admin-tools/pyenv-newest-versions @@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then echo "This script should be *sourced* rather than run directly through bash" exit 1 fi -export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.16' +export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.17' diff --git a/uncompyle6/scanners/tok.py b/uncompyle6/scanners/tok.py index 43d54cf4..986b9664 100644 --- a/uncompyle6/scanners/tok.py +++ b/uncompyle6/scanners/tok.py @@ -94,6 +94,7 @@ class Token: else: if version_tuple > (3, 9): print("Python versions 3.9 and greater are not supported.") + sys.exit(1) else: print(f"xdis might need to be informed about version {e}") return diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index c30ccf1d..431b6801 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -1,5 +1,5 @@ # -# (C) Copyright 2015-2018, 2020-2021 by Rocky Bernstein +# (C) Copyright 2015-2018, 2020-2021, 2023 by Rocky Bernstein # (C) Copyright 2000-2002 by hartmut Goebel # # This program is free software: you can redistribute it and/or modify From 36f00d334ed3cd273ca07e75daa8bc2fb56c3540 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 16 Jun 2023 07:10:37 -0400 Subject: [PATCH 09/11] Revert last change. --- uncompyle6/scanners/tok.py | 1 - 1 file changed, 1 deletion(-) diff --git a/uncompyle6/scanners/tok.py b/uncompyle6/scanners/tok.py index 986b9664..43d54cf4 100644 --- a/uncompyle6/scanners/tok.py +++ b/uncompyle6/scanners/tok.py @@ -94,7 +94,6 @@ class Token: else: if version_tuple > (3, 9): print("Python versions 3.9 and greater are not supported.") - sys.exit(1) else: print(f"xdis might need to be informed about version {e}") return From 568b64b59ec6c422787c3ef9655ba3ed9a718f2d Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 16 Jun 2023 07:30:25 -0400 Subject: [PATCH 10/11] Allow decompilation of older bytecode from 3.9+ --- uncompyle6/bin/uncompile.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index d5beaaad..c026db88 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -80,30 +80,7 @@ def usage(): def main_bin(): - if not ( - sys.version_info[0:2] - in ( - (2, 6), - (2, 7), - (3, 0), - (3, 1), - (3, 2), - (3, 3), - (3, 4), - (3, 5), - (3, 6), - (3, 7), - (3, 8), - (3, 9), - (3, 10), - ) - ): - print( - f"Error: {program} can decompile only bytecode from Python 3.7" - f""" to 3.8.\n\tYou have version: {version_tuple_to_str()}.""" - ) - sys.exit(-1) - + current_bytecode_supported = True recurse_dirs = False numproc = 0 outfile = "-" From 828b1c989d90b8a27bd6c18da2fa5cf567caa2fb Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 29 Jun 2023 15:56:53 -0400 Subject: [PATCH 11/11] Fix fragment bugs mostly with respect to show_ast handling --- uncompyle6/semantics/fragments.py | 22 +++++++++++++++------- uncompyle6/show.py | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index dc4e8e47..1c25bb71 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2019, 2021-2022 by Rocky Bernstein +# Copyright (c) 2015-2019, 2021-2023 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1169,11 +1169,14 @@ class FragmentsWalker(pysource.SourceWalker, object): self.p.insts = self.scanner.insts self.p.offset2inst_index = self.scanner.offset2inst_index ast = python_parser.parse(self.p, tokens, customize, code) + self.customize(customize) self.p.insts = p_insts + except (python_parser.ParserError, AssertionError) as e: raise ParserError(e, tokens) + transform_tree = self.treeTransform.transform(ast, code) maybe_show_tree(self, ast) - return ast + return transform_tree # The bytecode for the end of the main routine has a # "return None". However you can't issue a "return" statement in @@ -1199,23 +1202,28 @@ class FragmentsWalker(pysource.SourceWalker, object): if len(tokens) == 0: return PASS - # Build parse tree from tokenized and massaged disassembly. + # Build a parse tree from tokenized and massaged disassembly. try: # FIXME: have p.insts update in a better way # modularity is broken here p_insts = self.p.insts self.p.insts = self.scanner.insts self.p.offset2inst_index = self.scanner.offset2inst_index + self.p.opc = self.scanner.opc ast = parser.parse(self.p, tokens, customize, code) self.p.insts = p_insts - except (parser.ParserError, AssertionError) as e: + except (python_parser.ParserError, AssertionError) as e: raise ParserError(e, tokens, {}) - maybe_show_tree(self, ast) - checker(ast, False, self.ast_errors) - return ast + self.customize(customize) + transform_tree = self.treeTransform.transform(ast, code) + + maybe_show_tree(self, ast) + + del ast # Save memory + return transform_tree # FIXME: we could provide another customized routine # that fixes up parents along a particular path to a node that diff --git a/uncompyle6/show.py b/uncompyle6/show.py index 52b8dd5e..376e810e 100644 --- a/uncompyle6/show.py +++ b/uncompyle6/show.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018, 2020 Rocky Bernstein +# Copyright (C) 2018, 2020, 2023 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by