From bbaa3e660221956219a7bbe32c68944514a776f2 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 07:50:22 -0500 Subject: [PATCH 1/7] Fix bug in single param **arg --- test/bytecode_2.1/02_def.pyc | Bin 0 -> 950 bytes test/bytecode_2.2/02_def.pyc | Bin 0 -> 950 bytes test/bytecode_2.3/02_def.pyc | Bin 802 -> 963 bytes test/bytecode_2.4/02_def.pyc | Bin 642 -> 754 bytes test/bytecode_2.5/02_def.pyc | Bin 759 -> 893 bytes test/bytecode_2.6/02_def.pyc | Bin 759 -> 893 bytes test/bytecode_2.7/02_def.pyc | Bin 0 -> 893 bytes test/bytecode_3.2/02_def.pyc | Bin 810 -> 945 bytes test/bytecode_3.3/02_def.pyc | Bin 864 -> 1016 bytes test/bytecode_3.4/02_def.pyc | Bin 626 -> 733 bytes test/bytecode_3.5/02_def.pyc | Bin 628 -> 733 bytes test/simple_source/def/02_def.py | 6 ++++++ uncompyle6/semantics/make_function.py | 3 +-- 13 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 test/bytecode_2.1/02_def.pyc create mode 100644 test/bytecode_2.2/02_def.pyc create mode 100644 test/bytecode_2.7/02_def.pyc diff --git a/test/bytecode_2.1/02_def.pyc b/test/bytecode_2.1/02_def.pyc new file mode 100644 index 0000000000000000000000000000000000000000..51bfe0948516892522823a5098182ac020b865ba GIT binary patch literal 950 zcmcJNJ#WG=5QbmJfrgKUg-+co)S*CN?>}H+saOz=gM|8pFsSWNd12^(?VsrDO9Kk+ zP^30a?tG5#9-r?nKThj!?-!F4d;)&7XQqZt-EaE3+kv-^>>#yQm*>BN^&*G)zFswbFb1;u}?yM#psn{BuAXq Scjz@6+uwgnH^Y|ZsD1)gk)PcF literal 0 HcmV?d00001 diff --git a/test/bytecode_2.2/02_def.pyc b/test/bytecode_2.2/02_def.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ccd6f1cb59d84051a44a8a89d551c31244fb6c8a GIT binary patch literal 950 zcmcJN&rZTX5XQgl7AzFY1y9}$7X=D;-++r3!-ZfM65>C?LezuV3kP4;cktV33Q|p! zM7!*4|0XD#h^AJYKT){;!0gT2*VHCi{5{43L zxJ1H=gdtoDrYeaQm?K=ZBNi8wf`+Yo6ic!m_)#$&wrNl@_LT4J{C;sa9oxtCGMjeH zbk^ynW4epQi}gSL+*Oc?R+)ky1%dxTe2L&;)|<%=_B&25EI9KD^GmP@uJ(tpG%8@3 zrl6gu7_RY!qqm|&+Kt%kP=Wn;%M66cSdJ=zYyJ~+kZhTd!y_}H8g`I75$3lC7;?$& zld%U#tTAP_e#j=P)zY>K>abn)dzUv-?s~aPbJx$^!0zRL%GzJo-K$Z~A}a6kA3M%> R=p`H5?|)M_!lve^egGf2pW*-j literal 0 HcmV?d00001 diff --git a/test/bytecode_2.3/02_def.pyc b/test/bytecode_2.3/02_def.pyc index 8480c67f198cf94727442775b50b987a7d50a5c4..ba1a28764adc8e2cbe32728a75330f53ba9b7082 100644 GIT binary patch delta 197 zcmZ3)c9>n-`V%i#ZHHP^G6NJa0ci&yE_R+MZNt;Tz!1gCkiyOotiiD{%8bbztd0>V z55nFcbsRtN_+J+}sgB{3*NKRhQsIakzmx%R^K diff --git a/test/bytecode_2.4/02_def.pyc b/test/bytecode_2.4/02_def.pyc index 1e44dda520b0581f3f664997bbbf4999793cb510..ed8fbab09174c1bf222ae231bf2470b82dde06d0 100644 GIT binary patch delta 179 zcmZo-{lqGr`-zvUwnHr{nE?uzfV2Y;7duasw&7`EV2EO6NMUCP*5KF}b(E0}td4Q= zX-0(vMxd$^79f#bo>HsOk8f-u+2&6ZN k8AN~#2x12j93X-VNI;ZuO?F^1;{)+InK@Z_n1PTD0IA0r0{{R3 delta 94 zcmeyw+QcfI`-zw9eRE}4G6NJa0ci&yF1DH|ZNnU_!M-v1D5IjE1}ji3h!I3Eg9sK7 b!44ugfP@C?NWT>uU! diff --git a/test/bytecode_2.5/02_def.pyc b/test/bytecode_2.5/02_def.pyc index cd98701ca37892b2a4013d8e962efb3fa91f52fe..fcd3d2e78af24895ba121c2deeaa161ee7ecc16a 100644 GIT binary patch delta 138 zcmey)`j<_5^Cw=e+77j-WCkc;0@4mZTivfgy^OA%&eGSc79@lpzxvSRLbJ z872i@Mxd$^79f#bo>Dy7pGjMq6Ua#`%}Y)!DJd!z0T}`^je*fmgKcsjlMGW3=frxW V$v2sV__%<=oXngoJj_7I1^{Jq83q6V delta 81 zcmey%_MKIF^Cw=e+_ysE$qZ1y1f(5+xY%livfgy^OA%&eGSc79@lpzxvSRLbJ z872i@Mxd$^79f#bo>Dy7pGjMq6Ua#`%}Y)!DJd!z0T}`^je*fmgKcsjlMGW3=fryB V$+wsU`M7|>oXngoJj_7I1^{&*86^M! delta 81 zcmey%_MKJw;wN6N_sx}I$qZ1y1f(5+xY%lcAu%F#aJMW~u~Vs{rB$5>7}z@SfB6I4d*nkx zL^Mh6^7`Vt$M!{ioeVxcFH%G0)#LX)n|okqiS|S(3r~bcwntXc-lLQSKfY)sIuPxX z4P|mbwolU};yQJ8j|V91{GaO61d&O?j($0J&M~)KZ&%sOZHk@AhBjLaN8=g4F1PPF zR{AzVzJnysKv&jE!Gs)c-T)6{U18BBYZP~N98Fi~%_)kPePmhpu@x$P1n>FPx(>LQQx?UGndkY= yo$;WgPLhxZnvFEpp=DPR>tl$#1|#a<-zMUm+x5m4tLzTzbpv*>iv457clZtCFodxH literal 0 HcmV?d00001 diff --git a/test/bytecode_3.2/02_def.pyc b/test/bytecode_3.2/02_def.pyc index aab6b55b1888642e55042b25355a9a9128e56085..9677152a6ce1566bbaad6ff6aa5d528c97d6df4e 100644 GIT binary patch delta 266 zcmZ3*wvk;vhliJ|wnHr{nE?qf0oe{fT}e-vddG7 zH9#uBL@CTpX-=R}T4`Q#Vo6C+u?Ua@HOEhbZSqzoHKkG%1qc&RjM3%e0_x;s=49bv H20}Ifc%Lkc delta 189 zcmdnUzKTsghliKzeRE}4G6ND|0fS&&39!AB9C%ln( zlPwqn5Znp`Q*p8;Q#cZr--HijCdhCGMn4VK$$HFcVj3VGf+&UCA|uNIl(EUpPbtkw TwPOUbi$MaM%$zJd%sk8hR1O{e diff --git a/test/bytecode_3.3/02_def.pyc b/test/bytecode_3.3/02_def.pyc index 8d84c270848dc73eb6f1eb5bf35b1f10de2defe0..860db3c9b54f0a2a18b36874a1b7ccb2118d4670 100644 GIT binary patch delta 273 zcmaFB_Jdt{9uF^9ZHHRa4+aK?WFUqEW+2M}h>JrfDtq&%@G!J6FhsF3r0_BXYw%4h zv|*73N>0{iQk-}t2+p4D%NUN}oMw!Jaq3|V#mQHhgt_20F;329QkdM&$STePG%mY5 zrC0+MgKhF_CN(8gQG|J@hE8^7 S7Ubgsn!>}(!@|!Dglqt{s4xKl delta 172 zcmeyt{(wz+9uF@UYfNQW4Fdy1G7!T7Gmzx~#KoQymA#pRHFzf0+OWw21@!bMZ(vlM zcrOTvH#w9s9Ko$eFcl|JZZDtj{rYw~QY-Nwl5r^!0Gk4b9s TS|%kP4xlU#GY<JrfDtq&%@G!J6FhsF3r0_BXYw}Gj zG-j0ExV(vxD;cJNaq>+@Srf)vEZOBL#YK!jVFJZZDtj{rYw}F2HD;F8)8DwUiILe) alXY?*lhow3OiDZ)Kt(*vJS_ao{LBE)@(+6e diff --git a/test/simple_source/def/02_def.py b/test/simple_source/def/02_def.py index 003ad84a..09f1a002 100644 --- a/test/simple_source/def/02_def.py +++ b/test/simple_source/def/02_def.py @@ -13,3 +13,9 @@ def x3(a, b, c=5): def x4(a, b=5, **c): pass + +# Had a bug in 2.x where +# we weren't picking up **kwds when +# it was the sole parameter +def funcattrs(**kwds): + return diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 8a1d1f4a..7cfeadae 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -392,8 +392,7 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None): if code_has_star_star_arg(code): if argc > 0: self.write(', ') - if argc + kw_pairs > 0: - self.write('**%s' % code.co_varnames[argc + kw_pairs]) + self.write('**%s' % code.co_varnames[argc + kw_pairs]) if is_lambda: self.write(": ") From 9d77b5a95677355d3eca9f01daa5b6f91d8c2f82 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 08:31:13 -0500 Subject: [PATCH 2/7] Bug in 2.6 tryelse rule --- uncompyle6/parsers/parse26.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index 2d40747c..c7e5ebc7 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -39,7 +39,7 @@ class Python26Parser(Python2Parser): except_handler tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK - except_handler else_suite COME_FROM + except_handler else_suite come_froms _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP From 2599b94786b21dcf92d18b5d7eac300c2cee5443 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 13:27:19 -0500 Subject: [PATCH 3/7] Start to handle FUTURE_UNICODE_LITERALS flag --- test/bytecode_2.6/05_unicode_literals.pyc | Bin 0 -> 257 bytes test/bytecode_2.7/05_unicode_literals.pyc | Bin 0 -> 257 bytes test/bytecode_3.0/05_unicode_literals.pyc | Bin 0 -> 278 bytes test/bytecode_3.1/05_unicode_literals.pyc | Bin 0 -> 278 bytes .../stmts/05_unicode_literals.py | 8 +++++ test/stdlib/runtests.sh | 5 +-- uncompyle6/semantics/pysource.py | 33 ++++++++++++++++-- 7 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 test/bytecode_2.6/05_unicode_literals.pyc create mode 100644 test/bytecode_2.7/05_unicode_literals.pyc create mode 100644 test/bytecode_3.0/05_unicode_literals.pyc create mode 100644 test/bytecode_3.1/05_unicode_literals.pyc create mode 100644 test/simple_source/stmts/05_unicode_literals.py diff --git a/test/bytecode_2.6/05_unicode_literals.pyc b/test/bytecode_2.6/05_unicode_literals.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e843727d35d6b4ee97b96fead14fb0575628f0d GIT binary patch literal 257 zcmZ8bOA5j;5S{dg6oj6@r34q&jXM!f;G*ub32hT0nA*}z79P_(cmgk=lOhxxhWFmg zJZ9$kUgoRUTD0(snD_({xHSny$>0WH83+bUMIa(jF^Cvc0wMvG!i8sKH6IGjNMS+X z@22gQme!yiYwLWN)Hq{v=4mkJ1X5l-`cX?+Q3chsB_h?-I8nnxe+IXq(wT0sTDsnk itu7sQ=*rz*{$Jh>4_uJ_(Dlk&eGJQ8__#i3l0_@o8PPkzwk%sJkK@r5ccL?PZEs5jvN literal 0 HcmV?d00001 diff --git a/test/bytecode_3.1/05_unicode_literals.pyc b/test/bytecode_3.1/05_unicode_literals.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f35c74e824e880aa8a2784fe30c34249e57d41b1 GIT binary patch literal 278 zcmaKn%?iRW49C-bAwywL;8BJL=f#_dPoM{Hy_9XOP-xv)nuG7_gP7uB;=z#ol8}EO zxh_}b`n8cgd|Ar%nFBWr%`U(JcmYy?&i3l0_@o8PkABeP%qiZ4@r5ccL?PZaU^n#u literal 0 HcmV?d00001 diff --git a/test/simple_source/stmts/05_unicode_literals.py b/test/simple_source/stmts/05_unicode_literals.py new file mode 100644 index 00000000..a6215597 --- /dev/null +++ b/test/simple_source/stmts/05_unicode_literals.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +# __future__ unicode_literals changes the way we need to print +# the below +# In Python assembler code "a" is u"a" and b"a" is "a". +a = "a" +ba = b"a" +bb = b"b" diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index e0d58906..7350a0b2 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -73,12 +73,13 @@ case $PYVERSION in SKIP_TESTS=( [test_builtin.py]=1 [test_contextlib.py]=1 # decorators - [test_decorators.py]=1 # decorators [test_descr.py]=1 # syntax error look at [test_dis.py]=1 # We change line numbers - duh! [test_future4.py]=1 # Possible additional rule for future mechanism? [test_grammar.py]=1 # Too many stmts. Handle large stmts [test_importlib.py]=1 # Control flow? + [test_ioctl.py]=1 # Test takes too long to run + [test_itertools.py]=1 # Syntax error - look at! ) ;; *) @@ -108,7 +109,7 @@ if [[ -n $1 ]] ; then files=$1 SKIP_TESTS=() else - files=test_*.py + files=test_[m]*.py fi for file in $files; do [[ -v SKIP_TESTS[$file] ]] && continue diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 9ff485e5..9c76197f 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -214,6 +214,10 @@ class SourceWalker(GenericASTTraversal, object): self.line_number = 0 self.ast_errors = [] + # This is in Python 2.6 on. It changes the way + # strings get interpreted. See n_LOAD_CONST + self.FUTURE_UNICODE_LITERALS = False + # Sometimes we may want to continue decompiling when there are errors # and sometimes not self.tolerate_errors = tolerate_errors @@ -644,7 +648,8 @@ class SourceWalker(GenericASTTraversal, object): if self.pending_newlines: out = out[:-self.pending_newlines] - if isinstance(out, str) and not PYTHON3: + if (isinstance(out, str) and + not (PYTHON3 and self.FUTURE_UNICODE_LITERALS)): out = unicode(out, 'utf-8') self.f.write(out) @@ -843,6 +848,27 @@ class SourceWalker(GenericASTTraversal, object): self.write('None') elif isinstance(data, tuple): self.pp_tuple(data) + elif self.FUTURE_UNICODE_LITERALS: + # The FUTURE_UNICODE_LITERALS compiler flag + # in 2.6 on change the way + # strings are interpreted: + # u'xxx' -> 'xxx' + # xxx' -> b'xxx' + if isinstance(data, unicode): + try: + try: + data = str(data) + except UnicodeEncodeError: + # Have to keep data as it is: in Unicode. + pass + self.write(repr(data)) + except: + from trepan.api import debug; debug() + self.write(repr(data)) + elif isinstance(data, str): + self.write('b'+repr(data)) + else: + self.write(repr(data)) else: self.write(repr(data)) # LOAD_CONST is a terminal, so stop processing/recursing early @@ -1592,7 +1618,7 @@ class SourceWalker(GenericASTTraversal, object): n_classdefdeco2 = n_classdef def print_super_classes(self, node): - if not (node == 'list'): + if not (node == 'tuple'): return n_subclasses = len(node[:-1]) @@ -2378,6 +2404,9 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False, except: pass + deparsed.FUTURE_UNICODE_LITERALS = ( + COMPILER_FLAG_BIT['FUTURE_UNICODE_LITERALS'] & co.co_flags != 0) + # What we've been waiting for: Generate source from AST! deparsed.gen_source(deparsed.ast, co.co_name, customize) From ee3bdbc2ed87ec478022c74c96bbc8261f59fb6a Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 14:12:03 -0500 Subject: [PATCH 4/7] logic bug: and->or --- uncompyle6/semantics/pysource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 9c76197f..a611ae1b 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -649,7 +649,7 @@ class SourceWalker(GenericASTTraversal, object): if self.pending_newlines: out = out[:-self.pending_newlines] if (isinstance(out, str) and - not (PYTHON3 and self.FUTURE_UNICODE_LITERALS)): + not (PYTHON3 or self.FUTURE_UNICODE_LITERALS)): out = unicode(out, 'utf-8') self.f.write(out) From e3f62e4a1a6d945fe8b38d050f3b93364828eb58 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 14:20:01 -0500 Subject: [PATCH 5/7] unicode bug fix try #2... this time, for sure! --- uncompyle6/semantics/pysource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index a611ae1b..b9551157 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -650,6 +650,7 @@ class SourceWalker(GenericASTTraversal, object): out = out[:-self.pending_newlines] if (isinstance(out, str) and not (PYTHON3 or self.FUTURE_UNICODE_LITERALS)): + from trepan.api import debug; debug() out = unicode(out, 'utf-8') self.f.write(out) @@ -854,7 +855,7 @@ class SourceWalker(GenericASTTraversal, object): # strings are interpreted: # u'xxx' -> 'xxx' # xxx' -> b'xxx' - if isinstance(data, unicode): + if not PYTHON3 and isinstance(data, unicode): try: try: data = str(data) From f07dcb1508c7a08e6dfb01ecfcdc1dfdc658c0ac Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 14:23:05 -0500 Subject: [PATCH 6/7] Remove debug schmutz --- uncompyle6/semantics/pysource.py | 1 - 1 file changed, 1 deletion(-) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index b9551157..1b2a327a 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -650,7 +650,6 @@ class SourceWalker(GenericASTTraversal, object): out = out[:-self.pending_newlines] if (isinstance(out, str) and not (PYTHON3 or self.FUTURE_UNICODE_LITERALS)): - from trepan.api import debug; debug() out = unicode(out, 'utf-8') self.f.write(out) From 7c58f8b41db105bbecd4d4adfae1942f36c0032d Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 5 Dec 2017 18:13:22 -0500 Subject: [PATCH 7/7] runtest.sh: remove from exlusion stdlib test that now work --- test/stdlib/runtests.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 7350a0b2..45f51132 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -55,15 +55,13 @@ case $PYVERSION in [test_codecs.py]=1 # need to fix tryelse [test_coercion.py]=1 # Control flow? [test_cookielib.py]=1 # Control flow? - [test_contextlib.py]=1 # decorators - [test_decorators.py]=1 # decorators + [test_decorators.py]=1 # Syntax Error - look at [test_enumerate.py]=1 # Control flow? [test_file.py]=1 # Control flow? [test_format.py]=1 # Control flow? [test_frozen.py]=1 # Control flow? [test_ftplib.py]=1 # Control flow? [test_funcattrs.py]=1 # Control flow? - [test_future4.py]=1 # Possible additional rule for future mechanism? [test_grp.py]=1 # Long test - might work Control flow? [test_pwd.py]=1 # Long test - might work? Control flow? [test_queue.py]=1 # Control flow? @@ -71,13 +69,9 @@ case $PYVERSION in ;; 2.7) SKIP_TESTS=( - [test_builtin.py]=1 - [test_contextlib.py]=1 # decorators - [test_descr.py]=1 # syntax error look at + [test_builtin.py]=1 # Syntax error, look at [test_dis.py]=1 # We change line numbers - duh! - [test_future4.py]=1 # Possible additional rule for future mechanism? [test_grammar.py]=1 # Too many stmts. Handle large stmts - [test_importlib.py]=1 # Control flow? [test_ioctl.py]=1 # Test takes too long to run [test_itertools.py]=1 # Syntax error - look at! )