From 9e34654b385759ee52ca4b5dca9a6006cae3dffe Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 10 Apr 2017 02:47:46 -0400 Subject: [PATCH 01/11] Add more while1else grammar rules Towards addressing issue #93 --- test/bytecode_2.7/02_while1else.pyc | Bin 0 -> 301 bytes test/bytecode_2.7/09_whiletrue_bug.pyc | Bin 320 -> 503 bytes test/bytecode_3.3/02_while1else.pyc | Bin 0 -> 340 bytes test/bytecode_3.5/02_while1else.pyc | Bin 0 -> 255 bytes test/bytecode_3.5/09_whiletrue_bug.pyc | Bin 289 -> 435 bytes test/simple_source/stmts/02_while1else.py | 8 ++++++++ uncompyle6/parsers/parse2.py | 4 ++-- uncompyle6/parsers/parse3.py | 2 ++ uncompyle6/parsers/parse35.py | 6 ++++-- 9 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 test/bytecode_2.7/02_while1else.pyc create mode 100644 test/bytecode_3.3/02_while1else.pyc create mode 100644 test/bytecode_3.5/02_while1else.pyc create mode 100644 test/simple_source/stmts/02_while1else.py diff --git a/test/bytecode_2.7/02_while1else.pyc b/test/bytecode_2.7/02_while1else.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecf6378764b759bb18992d69a2d25f863582dc3e GIT binary patch literal 301 zcma(~I|{-;5S?`kC6LY|Sj5JtwTO^Dh!%=XkYvDxjft5Je!xb&k;k(10?tIi)`fX9 z?EC4L?>+j@bqe37c+YsqXUG8V02@9OEQ1#Rq{7P~JK~`(!^s!LRlz~S>el!^^4N$r z;DN^O+I(%ZtU@9u5tk1q0_q8vytJ55QCBI3#^MMq;h;P_kA-tnVq^{O kofcb%0hCr=ORG4LDN^%;MBckVrC!U;<(g?F_`lB0!>omm!6bp_Y-MkeMO838;esXbMP`UnYpu0AdC( zA8cM?N=i{`aWUA2Du^5d!^BtmN?br?0htA<&N-Q>c_o5CUNHwqD xc_kp16>|Uy21aH^P9|nXGlI}`8uMHfq~&M5W@izkmUfx#R5Pg1&A0KS{N9jfD9&vU`^&^MxZbVfYgE5 z&Olr&0wgMUL27Fm848&h!kfT)gEg7_UV;>^WYA=aVoyv-DM~FazQtHo1QG-jzf_7d za|?1($BKb5adcbBa^-3My}L1!NYaI_G4j=9MsmEaU(Z p42;Z-TujW2MXW%!pC*Jxm<(aD0Of9R*yQG?l;)(`f%Ss40s!4WFJ%A# literal 0 HcmV?d00001 diff --git a/test/bytecode_3.5/09_whiletrue_bug.pyc b/test/bytecode_3.5/09_whiletrue_bug.pyc index 4c843d60823f204f1a15ee8fa9146200cd7b6d34..992ad7266946c523dc4e6d08e6319ce3e6741e59 100644 GIT binary patch literal 435 zcmYjMO-sW-6r8u4G$mTZQv3_jgVw81q#%M`im1?9521F~8f=<$H_=w`qW&5G(q27@ zXD^<7i3fc!!|bs0KDHW-@N4^XctXHC{?!`m9>bi5QWd=uO%9VQhWvJLR;JkMY<7Oxlkc0LWsR%49!jAy@}ej?;`k0ck9Ym z=*m&S8P3OW;LO=elK9BY(sux#qW{^FYaz0C0tOUN5LO<{8D8gDkHG}?pa{yt8xf^X zBLwVY6~inbhVpS6;706gQCdg%b2tPC5&@|Imy>Wq2X3Gmlb4pSzDWAcCy@(sQ&=}x zHo7eOeGu!lL$y3|u&Q52? m*~_E6=}gbreP0~5Yd4(hM)eh+{Pfx#!7ll-V@hb`Mf42`dPR@` diff --git a/test/simple_source/stmts/02_while1else.py b/test/simple_source/stmts/02_while1else.py new file mode 100644 index 00000000..a01efafa --- /dev/null +++ b/test/simple_source/stmts/02_while1else.py @@ -0,0 +1,8 @@ +# From Python-3.5.2/Lib/multiprocessing/connection.py + +def PipeClient(address): + while 1: + z = 2 + else: + raise + return diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 1de5e070..a76a970b 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -41,10 +41,10 @@ class Python2Parser(PythonParser): def p_stmt2(self, args): """ while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM - while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK COME_FROM while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK COME_FROM - while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index b5045bf9..633673ed 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -381,6 +381,8 @@ class Python3Parser(PythonParser): while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK + else_suite COME_FROM_LOOP # FIXME: investigate - can code really produce a NOP? whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 9e6cf248..0b1d4b2a 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -20,8 +20,10 @@ class Python35Parser(Python34Parser): # I'm sure by the time Python 4 comes around these will be turned # into special opcodes - while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK - POP_BLOCK COME_FROM_LOOP + while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK + POP_BLOCK COME_FROM_LOOP + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK + POP_BLOCK else_suite COME_FROM_LOOP # Python 3.5+ Await statement stmt ::= await_stmt From 41343c27b78073b6bf9b6f30c77e8305a8c256d7 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 10 Apr 2017 07:57:56 -0400 Subject: [PATCH 02/11] Misc bugs parse2.py: restore accidently-removed while1stmt rule scanner27.py: grammar typo check_ast: add while1else to list of looping constructs pysource.py: CALL_FUNCTION_VAR_KW_ARGS with positional args rule is different? --- uncompyle6/parsers/parse2.py | 1 + uncompyle6/scanners/scanner27.py | 4 ++-- uncompyle6/semantics/check_ast.py | 2 +- uncompyle6/semantics/pysource.py | 4 +++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index a76a970b..56ba9713 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -41,6 +41,7 @@ class Python2Parser(PythonParser): def p_stmt2(self, args): """ while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM + while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK COME_FROM while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK COME_FROM while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index 885be81e..0fab1c0f 100755 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, 2016 by Rocky Bernstein +# Copyright (c) 2015-2017 by Rocky Bernstein """ Python 2.7 bytecode ingester. @@ -48,7 +48,7 @@ class Scanner27(Scanner2): self.opc.DELETE_SLICE_2, self.opc.DELETE_SLICE_3, ]) - # opcodes with expect a variable number pushed values whose + # opcodes which expect a variable number pushed values and whose # count is in the opcode. For parsing we generally change the # opcode name to include that number. varargs_ops = set([ diff --git a/uncompyle6/semantics/check_ast.py b/uncompyle6/semantics/check_ast.py index 8bca0411..efea085d 100644 --- a/uncompyle6/semantics/check_ast.py +++ b/uncompyle6/semantics/check_ast.py @@ -10,7 +10,7 @@ before reduction and don't reduce when there is a problem. def checker(ast, in_loop, errors): in_loop = in_loop or ast.type in ('while1stmt', 'whileTruestmt', - 'whilestmt', 'whileelsestmt', + 'whilestmt', 'whileelsestmt', 'while1elsestmt', 'for_block') if ast.type in ('augassign1', 'augassign2') and ast[0][0] == 'and': text = str(ast) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index fba9e849..87bbf918 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1786,7 +1786,9 @@ class SourceWalker(GenericASTTraversal, object): str += '*%c, **%c)' # Python 3.5 only puts optional args (the VAR part) # lowest down the stack - if self.version == 3.5: + na = (v & 0xff) # positional parameters + if self.version == 3.5 and na == 0: + print("XXX", k) if p2[2]: p2 = (2, -2, ', ') entry = (str, 0, p2, 1, -2) else: From 6acec471e38a68a7b67f037d1c9382e2197f45af Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 11 Apr 2017 05:10:38 -0400 Subject: [PATCH 03/11] Towards fixing annotated decorator functions... and annotate functions --- test/bytecode_3.1/04_def_annotate.pyc | Bin 1300 -> 1758 bytes test/bytecode_3.4/04_def_annotate.pyc | Bin 974 -> 1315 bytes test/bytecode_3.5/04_def_annotate.pyc | Bin 974 -> 1315 bytes test/simple_source/bug31/04_def_annotate.py | 6 ++++++ uncompyle6/parsers/parse3.py | 6 +++--- uncompyle6/semantics/pysource.py | 1 - 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/bytecode_3.1/04_def_annotate.pyc b/test/bytecode_3.1/04_def_annotate.pyc index be4c0973f9e639eff0c1c0428aea6425f43da7d5..4529a695a0e3f7d1bf621b19c97ea1ed90547267 100644 GIT binary patch delta 495 zcmZ8e%SyvQ6upxfYa5M7wOUuDh!OQ=rd3bC=L;NFnDsv zY{Z~1Aac-#qtvHiTS3T`v3dH>lzJyvn%vQ(SE#!_GVr}(k~&(&x0ANmAI>9@@;)_E zuG6`L08ZDR!^Kt+q)dZENon1JBw1u*qDx4F8*TC+G>I~IcuwKa$xeBeS cf?ITlgEI9Y1=Pz3O;KSL*5nOdXLZ&VKQ|{^tpET3 delta 99 zcmcb|JB3T$pNE%Aoue=!nE?rK0@)5gTs&){ygaWELkb)TPxQ`Z19F2kL^fVr&CKE# hqQNm)oXvA`9h;93A5e<~3mYRFqXe@kvk;>YBLF-~59|N{ diff --git a/test/bytecode_3.4/04_def_annotate.pyc b/test/bytecode_3.4/04_def_annotate.pyc index 28fea02465cc3f5ee5ef51b8ebc6efc0d9007783..0acbc4ab7c236f3b1e408a48e717710201ab266c 100644 GIT binary patch delta 430 zcmYjMJxc>Y5PiFQAMs2?qLPH95F)}MSXv7fViSZka?Ntyt|(k`aqrRyk|O>9C17PI zf~}o}zac;1T3ZqP2hPO^xA5M)nPJ|{&fRCP`(3ShKTlt0?;Q9<>3O6N3F*f}_oZ{J z8cC2-FR$0NJsD;`GlC2vw)8Gyfg?g{5s_)%XFcO!0{ASqQXOAV2K9 ziYfXfz;8B6TlSazp6}!pDTm=eOPPC84iYtv3tg4+dK|{f&RTe$W+sfXfzB=yl?!q= z2W|2#ZC>0^tT!#9EyCj9Di>@~@hfJVinCkMsRb+Y)FfFFC9!GE9>wnFIY5PiFQAMs2?;ztsaLWl^5U}-H_h)odE$TiD(d!lg3#l1@-NQ(Falz^3; z2)1?>{)YU3Yi(`(2hPR_xA5M)nPJ|{&fOQUyIre!KhNLi9~}5X>3O7&2-)Xj_qB7P z8c~xs>ObarI*wWiXgJVKw5s_)%XFcs;0{E=3vN*Y<4EQ6LRN+c_J@~@hf_VinBY>sRb+YOs9DorHO9Mp2XhfI7oue=!l8J%gF#{6d0aG-(##UfLX1L;0PDpNHUIzs diff --git a/test/simple_source/bug31/04_def_annotate.py b/test/simple_source/bug31/04_def_annotate.py index a42e2a09..a397f73c 100644 --- a/test/simple_source/bug31/04_def_annotate.py +++ b/test/simple_source/bug31/04_def_annotate.py @@ -17,3 +17,9 @@ def div(a: dict(type=float, help='the dividend'), ) -> dict(type=float, help='the result of dividing a by b'): """Divide a by b""" return a / b + +class SupportsInt(_Protocol): + + @abstractmethod + def __int__(self) -> int: + pass diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 633673ed..a2262470 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -275,12 +275,15 @@ class Python3Parser(PythonParser): stmt ::= funcdef_annotate funcdef_annotate ::= mkfunc_annotate designator + mkfuncdeco0 ::= mkfunc_annotate + # This has the annotation value. # LOAD_NAME is used in an annotation type like # int, float, str annotate_arg ::= LOAD_NAME # LOAD_CONST is used in an annotation string annotate_arg ::= LOAD_CONST + annotate_arg ::= LOAD_GLOBAL # This stores the tuple of parameter names # that have been annotated @@ -709,9 +712,6 @@ class Python3Parser(PythonParser): ('pos_arg ' * args_pos, opname)) self.add_unique_rule(rule, opname, token.attr, customize) if opname.startswith('MAKE_FUNCTION_A'): - # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' % - # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) - self.add_unique_rule(rule, opname, token.attr, customize) if self.version >= 3.3: rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 87bbf918..6570f330 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1788,7 +1788,6 @@ class SourceWalker(GenericASTTraversal, object): # lowest down the stack na = (v & 0xff) # positional parameters if self.version == 3.5 and na == 0: - print("XXX", k) if p2[2]: p2 = (2, -2, ', ') entry = (str, 0, p2, 1, -2) else: From bb24df596d3fe6aebd05d83f338737c96e7ea6a2 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 11 Apr 2017 17:09:10 -0400 Subject: [PATCH 04/11] Bang on 3.x annotations --- test/bytecode_3.1/04_def_annotate.pyc | Bin 1758 -> 2449 bytes test/bytecode_3.4/04_def_annotate.pyc | Bin 1315 -> 1857 bytes test/bytecode_3.5/04_def_annotate.pyc | Bin 1315 -> 1857 bytes test/simple_source/bug31/04_def_annotate.py | 5 ++ uncompyle6/semantics/fragments.py | 8 ++- uncompyle6/semantics/make_function.py | 61 ++++++++++++++++---- 6 files changed, 60 insertions(+), 14 deletions(-) diff --git a/test/bytecode_3.1/04_def_annotate.pyc b/test/bytecode_3.1/04_def_annotate.pyc index 4529a695a0e3f7d1bf621b19c97ea1ed90547267..607ee87a70c191d75037be517495daed26d40912 100644 GIT binary patch delta 559 zcmaKmzfJ-{5XNWOyNe)K2y&nS&p*v2RyGz^24ak8Am&UoG>F`4KyoDPUWf)O?G4%5 zH?Xm@x3aMC1$+Z*TW8K-Y;dzPH#_%zznKqvoOSDIJ9Sw8JZZz9lf-%(L%xh%*LEQI zz4`I}^=dZ>w;UQN*bV044j{;Zn2lR$lKNiZ>HV1thJ-Rv@|Z>dI|87=AOP|-5)d$y zCh%XIlyvA}VpK3>5tG8pK*ic10rv)3Z(@e;aTL__D=)2K5qrr`EFVJ}m~fWwQKzPt zjDtODDIpatBuNY`rbIOKyw`83h@esp;%_HZ@qQ8cr0AW;1p)LN-&|k7Eqp ztL7tH)L*8{Y@N5N8#CyP!j7UfBYF_M(vo5-=hW&t859-_i&?C|b3DT`Y{hWQZzeQy A9{>OV delta 120 zcmbOze2-V&pNE(0h%r7*gO!bfR|^Pct(^6emb9SVL^% z^Bv5SIoJ;{icdbtzJ^hBav6s_BiH0<9A1-!I5R~AfTl>WurabRN-&Eu3o!~YDzX3o D_xl%a diff --git a/test/bytecode_3.4/04_def_annotate.pyc b/test/bytecode_3.4/04_def_annotate.pyc index 0acbc4ab7c236f3b1e408a48e717710201ab266c..376e257550c059e9029de488497dc7fe805f93ae 100644 GIT binary patch delta 722 zcmb7A%}(1u5T5mJ5)%T4AW=+GASA8i1L*^_swy~eYM=^LD_<;Q8yaeCX?786r7DM3 zm8xDqegvL?0}s%L=*8D^;Q-ITj8SM0?V+okPow$y_WSm%{$sX#)0nBhTEBk#QUd&f zKTe(U7Zm#EY4_sMK@C1j@NC$O9EcX-3oyo0Wwj1b1%__x{PMyYTa&#uB{6^}AR@FN zqQv4wPeLSVUKMjYu{Oieej1n~jJI|V;?RVG_V_SSm12WJFB2HL0?dSoI8GGveJWod zzJV+8NrfhnIglJdNKrB|8&`)ES`m<_p(_UY8`&8vLpmOChsQvCeEJt*eID}lM6I8t z{_*?cEKNT2pCwrsB>H(d;>x13mt=t{q&8T{QEFVPP&!WbaFN)NP2E9_vcfU1|3y4c z{fVTt|B^ItRlcoy7B1)CRmWyq4S`PnO@3X%rF#LTlu|{__erSlxAVSQ9peg*bQo3% zirHY7yab^+jLo|&!iU`HG5j`j&xKuTu+#FN8K1ESPJUuZ5eJZ6Bn%Q@Ny$tuDUtv(qd4OO pit0W diff --git a/test/bytecode_3.5/04_def_annotate.pyc b/test/bytecode_3.5/04_def_annotate.pyc index 6501af8e1b4b6ab3bf3b87388fb2fb4bf9d01d68..784db92665b2fb06c190a3dc1861ffdf67b6c7a8 100644 GIT binary patch delta 722 zcmb7A%}(1u5T5mJ5|aXlAf=e3(2!8c2f_oiswz12)D)B=Dqk#P8wfSFG`k2#EQfyG@B?3cNfSE86$C+ZjPvr~5 zH*h7cRcI2K1IZDD6eSb0adk+c6#F*8}1CD_e>t7pvyRa>ems-h+5WkWPXPdSwvtbm`m delta 213 zcmX@ex0p*=jF*?|Q13RNl(d%*+tQ&XB^&5UeS=@$Xiq z$y-><*lw{Dr{<(h)@PGo6rb$DrZhQ}RYm+3Ut&^mNl{{QNp5OMMt;gI=FGg3BI(H! z*d*K}f#!*V%wz!)JWR}tMIaGPjv@shb&EAECqJ>Ihy%zj5(Wvdq+}+S6iEP?QJnDs oMfoN9$@w`&l0c)xC(E$&iwFQ^q*>S)*%+mnC76X6g&0*>0OH0lZ2$lO diff --git a/test/simple_source/bug31/04_def_annotate.py b/test/simple_source/bug31/04_def_annotate.py index a397f73c..52942dd5 100644 --- a/test/simple_source/bug31/04_def_annotate.py +++ b/test/simple_source/bug31/04_def_annotate.py @@ -18,6 +18,11 @@ def div(a: dict(type=float, help='the dividend'), """Divide a by b""" return a / b +class TestSignatureObject(unittest.TestCase): + def test_signature_on_wkwonly(self): + def test(*, a:float, b:str) -> int: + pass + class SupportsInt(_Protocol): @abstractmethod diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index aaa9cb8d..48cd3cbf 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -79,7 +79,9 @@ from uncompyle6.semantics.consts import ( TABLE_DIRECT, escape, minint, MAP ) -from uncompyle6.semantics.make_function import find_all_globals, find_none +from uncompyle6.semantics.make_function import ( + find_all_globals, find_none, code_has_star_arg, code_has_star_star_arg +) if PYTHON3: from itertools import zip_longest @@ -1659,10 +1661,10 @@ class FragmentsWalker(pysource.SourceWalker, object): params.reverse() # back to correct order - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): params.append('*%s' % code.co_varnames[argc]) argc += 1 - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): params.append('**%s' % code.co_varnames[argc]) argc += 1 diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 402b9f35..0b545426 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, 2016 by Rocky Bernstein +# Copyright (c) 2015-2017 by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel """ All the crazy things we have to do to handle Python functions @@ -45,6 +45,17 @@ def find_none(node): return True return False +# FIXME: put this in xdis +def code_has_star_arg(code): + """Return True iff + the code object has a variable positional parameter (*args-like)""" + return (code.co_flags & 4) != 0 + +def code_has_star_star_arg(code): + """Return True iff + The code object has a variable keyword parameter (**kwargs-like).""" + return (code.co_flags & 8) != 0 + # FIXME: DRY the below code... def make_function3_annotate(self, node, isLambda, nested=1, @@ -87,7 +98,7 @@ def make_function3_annotate(self, node, isLambda, nested=1, l = -len(node) while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'): annotate_args[annotate_tup[i]] = (node[j][0].attr, - node[j][0] == 'LOAD_CONST') + node[j][0] in ('LOAD_CONST', 'LOAD_GLOBAL')) i -= 1 j -= 1 @@ -96,9 +107,12 @@ def make_function3_annotate(self, node, isLambda, nested=1, # positional args are before kwargs defparams = node[:args_node.attr[0]] pos_args, kw_args, annotate_argc = args_node.attr + if 'return' in annotate_args.keys(): + annotate_argc = len(annotate_args) - 1 else: defparams = node[:args_node.attr] kw_args = 0 + annotate_argc = 0 pass if 3.0 <= self.version <= 3.2: @@ -144,7 +158,7 @@ def make_function3_annotate(self, node, isLambda, nested=1, indent = ' ' * l line_number = self.line_number - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): self.write('*%s' % code.co_varnames[argc + kw_pairs]) argc += 1 @@ -186,10 +200,12 @@ def make_function3_annotate(self, node, isLambda, nested=1, else: suffix = ', ' + # self.println(indent, '#flags:\t', int(code.co_flags)) - if kw_args > 0: - if not (4 & code.co_flags): + if kw_args + annotate_argc > 0: + if not code_has_star_arg(code): if argc > 0: + self.write(", *, ") else: self.write("*, ") @@ -211,10 +227,32 @@ def make_function3_annotate(self, node, isLambda, nested=1, self.write(', ') i += 1 pass + elif n == 'annotate_arg': + from trepan.api import debug; debug() + pass + pass + annotate_args = [] + for n in node: + if n == 'annotate_arg': + annotate_args.append(n[0].attr) + elif n == 'annotate_tuple': + t = n[0].attr + if t[-1] == 'return': + t = t[0:-1] + annotate_args = annotate_args[:-1] + pass + last = len(annotate_args) - 1 + for i in range(len(annotate_args)): + self.write("%s: %s" % (annotate_args[i], t[i])) + if i < last: + self.write(', ') + pass + pass + break pass pass - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): if argc > 0: self.write(', ') self.write('**%s' % code.co_varnames[argc + kw_pairs]) @@ -297,6 +335,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): else: defparams = node[:args_node.attr] kw_args = 0 + annotate_argc = 0 pass lambda_index = None @@ -335,7 +374,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): name, default in zip_longest(paramnames, defparams, fillvalue=None)] params.reverse() # back to correct order - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): params.append('*%s' % code.co_varnames[argc]) argc += 1 @@ -363,7 +402,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): break pass - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): if argc > 0: self.write(', ') self.write('**%s' % code.co_varnames[argc + kw_pairs]) @@ -472,7 +511,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): name, default in zip_longest(paramnames, defparams, fillvalue=None)] params.reverse() # back to correct order - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): if self.version > 3.0: params.append('*%s' % code.co_varnames[argc + kw_pairs]) else: @@ -498,7 +537,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): indent = ' ' * l line_number = self.line_number - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): self.write('*%s' % code.co_varnames[argc + kw_pairs]) argc += 1 @@ -552,7 +591,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): pass pass - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): if argc > 0: self.write(', ') self.write('**%s' % code.co_varnames[argc + kw_pairs]) From 45bd8e4058e815543afbb96bb8343068bcc5c1cd Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 12 Apr 2017 04:50:22 -0400 Subject: [PATCH 05/11] Handle Python 2.4 "if 1...." --- uncompyle6/parsers/parse24.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/uncompyle6/parsers/parse24.py b/uncompyle6/parsers/parse24.py index 92a798a2..ac8ff71d 100644 --- a/uncompyle6/parsers/parse24.py +++ b/uncompyle6/parsers/parse24.py @@ -14,6 +14,15 @@ class Python24Parser(Python25Parser): def p_misc24(self, args): ''' + # Python 2.4 only adds something like the below for if 1: + # However we will just treat it as a noop (which of course messes up + # simple verify of bytecode. + # See also below in reduce_is_invalid where we check that the JUMP_FORWARD + # target matches the COME_FROM target + stmt ::= nop_stmt + nop_stmt ::= JUMP_FORWARD POP_TOP COME_FROM + + # 2.5+ has two LOAD_CONSTs, one for the number '.'s in a relative import # keep positions similar to simplify semantic actions @@ -37,6 +46,25 @@ class Python24Parser(Python25Parser): gen_comp_body ::= expr YIELD_VALUE ''' + def add_custom_rules(self, tokens, customize): + super(Python24Parser, self).add_custom_rules(tokens, customize) + if self.version == 2.4: + self.check_reduce['nop_stmt'] = 'tokens' + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + invalid = super(Python24Parser, + self).reduce_is_invalid(rule, ast, + tokens, first, last) + if invalid: + return invalid + + # FiXME: this code never gets called... + lhs = rule[0] + if lhs == 'nop_stmt': + return not int(tokens[first].pattr) == tokens[last].offset + + return False + class Python24ParserSingle(Python24Parser, PythonParserSingle): pass From be9194c22381ded65c335c436bcc393bb6f8e390 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 12 Apr 2017 20:12:41 -0400 Subject: [PATCH 06/11] annotate args type need to be expr's not constants --- uncompyle6/parsers/parse3.py | 3 +-- uncompyle6/semantics/make_function.py | 11 ++++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index a2262470..727786d7 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -282,8 +282,7 @@ class Python3Parser(PythonParser): # int, float, str annotate_arg ::= LOAD_NAME # LOAD_CONST is used in an annotation string - annotate_arg ::= LOAD_CONST - annotate_arg ::= LOAD_GLOBAL + annotate_arg ::= expr # This stores the tuple of parameter names # that have been annotated diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 0b545426..6b3ea5ad 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -97,8 +97,7 @@ def make_function3_annotate(self, node, isLambda, nested=1, j = annotate_last-1 l = -len(node) while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'): - annotate_args[annotate_tup[i]] = (node[j][0].attr, - node[j][0] in ('LOAD_CONST', 'LOAD_GLOBAL')) + annotate_args[annotate_tup[i]] = node[j][0] i -= 1 j -= 1 @@ -227,14 +226,11 @@ def make_function3_annotate(self, node, isLambda, nested=1, self.write(', ') i += 1 pass - elif n == 'annotate_arg': - from trepan.api import debug; debug() - pass pass annotate_args = [] for n in node: if n == 'annotate_arg': - annotate_args.append(n[0].attr) + annotate_args.append(n[0]) elif n == 'annotate_tuple': t = n[0].attr if t[-1] == 'return': @@ -243,7 +239,8 @@ def make_function3_annotate(self, node, isLambda, nested=1, pass last = len(annotate_args) - 1 for i in range(len(annotate_args)): - self.write("%s: %s" % (annotate_args[i], t[i])) + self.write("%s: " % (t[i])) + self.preorder(annotate_args[i]) if i < last: self.write(', ') pass From b4ded9282236070826a382a664f0db144cdeaacc Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 13 Apr 2017 01:14:49 -0400 Subject: [PATCH 07/11] Add Python 2.3 rule for "if 1: ..." Fully fixes #97 for Python 2.3. Python 2.4 was fixed in a previous commit. --- test/bytecode_2.3/03_if1.pyc | Bin 0 -> 489 bytes uncompyle6/parsers/parse23.py | 30 +++++++++++++++++++++++++++++- uncompyle6/semantics/pysource.py | 5 +++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 test/bytecode_2.3/03_if1.pyc diff --git a/test/bytecode_2.3/03_if1.pyc b/test/bytecode_2.3/03_if1.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de82a5856f7bc71f29ec003f33cd123fe7093dfd GIT binary patch literal 489 zcmb7Ay9&ZU5Zp_mMnUZCEbLMljirT(U?pi{5p06yAt!m^5zyYx)y6MyH?K+|9?aZp z=63cL?|#pIzHArD!Ejz7#26`7AQ=Ml7(4(D0G^aD7{m-uWw0~_Rh&GoghZgt1|d`= zbXp#uj&4leLe>zF?**BK9|pPfl11sBM}BcnA~7Q3Q5yP=NV2Qz>!<8&82WVX@L)Dc z?_z)oGVYGCw~l$V)J9EFlazI)zhbE9tYXl6Wqq?~^GnFOV+!6- Vh3^(Y8~h8kvZPPQ`{}T*`~jn&T(bZG literal 0 HcmV?d00001 diff --git a/uncompyle6/parsers/parse23.py b/uncompyle6/parsers/parse23.py index 160c4de1..b2072e16 100644 --- a/uncompyle6/parsers/parse23.py +++ b/uncompyle6/parsers/parse23.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Rocky Bernstein +# Copyright (c) 2016-2017 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock @@ -14,6 +14,17 @@ class Python23Parser(Python24Parser): def p_misc23(self, args): ''' + # Python 2.4 only adds something like the below for if 1: + # However we will just treat it as a noop (which of course messes up + # simple verify of bytecode. + # See also below in reduce_is_invalid where we check that the JUMP_FORWARD + # target matches the COME_FROM target + stmt ::= if1_stmt + if1_stmt ::= JUMP_FORWARD JUMP_IF_FALSE THEN POP_TOP COME_FROM + stmts + JUMP_FORWARD COME_FROM POP_TOP COME_FROM + + # Used to keep semantic positions the same across later versions # of Python _while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM @@ -33,6 +44,23 @@ class Python23Parser(Python24Parser): lc_body ::= LOAD_FAST expr LIST_APPEND ''' + def add_custom_rules(self, tokens, customize): + super(Python23Parser, self).add_custom_rules(tokens, customize) + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + invalid = super(Python24Parser, + self).reduce_is_invalid(rule, ast, + tokens, first, last) + if invalid: + return invalid + + # FiXME: this code never gets called... + lhs = rule[0] + if lhs == 'nop_stmt': + return not int(tokens[first].pattr) == tokens[last].offset + + return False + class Python23ParserSingle(Python23Parser, PythonParserSingle): pass diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 6570f330..cc70aaf6 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -218,6 +218,11 @@ class SourceWalker(GenericASTTraversal, object): 'importlist2': ( '%C', (0, maxint, ', ') ), }) if version <= 2.4: + if version == 2.3: + TABLE_DIRECT.update({ + 'if1_stmt': ( '%|if 1\n%+%c%-', 5 ) + }) + global NAME_MODULE NAME_MODULE = AST('stmt', [ AST('assign', From 495bdd7b64fd686dcee6fbc0969ab9bfdcad4e04 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 13 Apr 2017 01:48:17 -0400 Subject: [PATCH 08/11] Towards fixing issue #92 --- uncompyle6/parsers/parse35.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 0b1d4b2a..2653d888 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -22,6 +22,7 @@ class Python35Parser(Python34Parser): while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP + while1stmt ::= SETUP_LOOP l_stmts POP_BLOCK COME_FROM_LOOP while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM_LOOP From a9f7a3c6d041ccda2c343b6ffa0bf2f6c2b009b3 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 13 Apr 2017 20:27:02 -0400 Subject: [PATCH 09/11] In 3.x come_from should include COME_FROM_EXCEPT --- uncompyle6/parsers/parse3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 727786d7..b011f064 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -156,7 +156,7 @@ class Python3Parser(PythonParser): # of missing "else" clauses. Therefore we include grammar # rules with and without ELSE. - ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM + ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec From 8cdaac93abc76d109f88f2e02d006f0cfb87a88a Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 13 Apr 2017 21:27:22 -0400 Subject: [PATCH 10/11] Add if1else. Fixes #101 --- test/bytecode_3.1/03_if_true_else.pyc | Bin 0 -> 174 bytes test/simple_source/bug27+/03_if_true_else.py | 1 + uncompyle6/parser.py | 6 +++++- uncompyle6/parsers/parse3.py | 1 - uncompyle6/semantics/consts.py | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 test/bytecode_3.1/03_if_true_else.pyc create mode 100644 test/simple_source/bug27+/03_if_true_else.py diff --git a/test/bytecode_3.1/03_if_true_else.pyc b/test/bytecode_3.1/03_if_true_else.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5085777c7c71da6620e4dac8117ba1534dfd4c9d GIT binary patch literal 174 zcmeb0;pK`|{t%JOfCLzUYzH7N76B3|ObkUl3@Ho@dCUx{j0}t^j10jV%s`1ukTSmz z4JIH}$^s-pib_*UIe?7#__WNN)cAM}Ae#X}l&S-{#hJMUK)K@l(xT*4{iM=#BXezi l1LOG2wD=OB{`l0K;#9qYO0YRLx%nxjIjMFai;6*J0RScCAkP2* literal 0 HcmV?d00001 diff --git a/test/simple_source/bug27+/03_if_true_else.py b/test/simple_source/bug27+/03_if_true_else.py new file mode 100644 index 00000000..bfdcae3c --- /dev/null +++ b/test/simple_source/bug27+/03_if_true_else.py @@ -0,0 +1 @@ +1 if 1 else __file__ diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 5169d550..9ab80c95 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -508,8 +508,12 @@ class PythonParser(GenericASTBuilder): expr ::= conditional conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr + expr ::= conditionalnot - conditionalnot ::= expr jmp_true expr _jump expr COME_FROM + conditionalnot ::= expr jmp_true expr _jump expr COME_FROM + + expr ::= conditionalTrue + conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM ret_expr ::= expr ret_expr ::= ret_and diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index b011f064..1c530bc2 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -409,7 +409,6 @@ class Python3Parser(PythonParser): conditional ::= expr jmp_false expr jf_else expr COME_FROM conditionalnot ::= expr jmp_true expr jf_else expr COME_FROM - expr ::= LOAD_CLASSNAME # Python 3.4+ diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index ee233ae3..6570522b 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -170,6 +170,7 @@ TABLE_DIRECT = { 'or': ( '%c or %c', 0, 2 ), 'ret_or': ( '%c or %c', 0, 2 ), 'conditional': ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)), + 'conditionalTrue': ( '%p if 1 else %p', (0, 27), (2, 27)), 'ret_cond': ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)), 'conditionalnot': ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)), 'ret_cond_not': ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)), From 39b9810587ebea18d95e5afffda8637c1b56716d Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 14 Apr 2017 05:05:02 -0400 Subject: [PATCH 11/11] Better names for a test --- .../{03_if_true_else.pyc => 03_if_1_else.pyc} | Bin .../bug27+/{03_if_true_else.py => 03_if_1_else.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/bytecode_3.1/{03_if_true_else.pyc => 03_if_1_else.pyc} (100%) rename test/simple_source/bug27+/{03_if_true_else.py => 03_if_1_else.py} (100%) diff --git a/test/bytecode_3.1/03_if_true_else.pyc b/test/bytecode_3.1/03_if_1_else.pyc similarity index 100% rename from test/bytecode_3.1/03_if_true_else.pyc rename to test/bytecode_3.1/03_if_1_else.pyc diff --git a/test/simple_source/bug27+/03_if_true_else.py b/test/simple_source/bug27+/03_if_1_else.py similarity index 100% rename from test/simple_source/bug27+/03_if_true_else.py rename to test/simple_source/bug27+/03_if_1_else.py