From 616e5c82f662ff358186ae1c910efe16c5621d9f Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 19 Sep 2018 20:09:22 -0400 Subject: [PATCH] Reinstat expr32 and expr1024 rules... to speed up handling long literal lists. See also issue #188 Update issue forms to simplfy via putting instructions as comments. --- .github/ISSUE_TEMPLATE/bug-report.md | 22 ++++++++++++------ .github/ISSUE_TEMPLATE/feature-request.md | 11 +++++---- test/bytecode_1.3/test_grammar.pyc | Bin 13010 -> 0 bytes test/simple_source/expression/06_huge_list.py | 4 ++++ uncompyle6/parser.py | 5 ++++ uncompyle6/parsers/parse2.py | 16 +++++++++++-- uncompyle6/parsers/parse3.py | 17 ++++++++++++-- uncompyle6/scanners/scanner2.py | 11 +++++++++ uncompyle6/semantics/helper.py | 2 ++ 9 files changed, 72 insertions(+), 16 deletions(-) delete mode 100644 test/bytecode_1.3/test_grammar.pyc diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 0793c448..0f2a61e7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -4,11 +4,11 @@ about: Tell us about uncompyle6 bugs --- -__Note:__ Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ? + + ## Description -Replace this text with a clear and concise description of the bug. + ## How to Reproduce -Please show both the input you gave and the + + ## Expected behavior -A clear and concise description of what you expected to happen. + ## Environment -_This section sometimes is optional but helpful to us._ + + ## Additional Environment or Context -_This section is optional._ + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index cab53e3b..3cbb75be 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -6,16 +6,17 @@ about: Tell us about a new feature that you would like to see in uncompyle6 ## Description -Replace this text with a short description of the feature. This might -include same input and output. + ## Background -Replace this text with any additional background for the -feature, for example: user scenarios, or the value of the feature. + ## Tests -_This section is optional._ + diff --git a/test/bytecode_1.3/test_grammar.pyc b/test/bytecode_1.3/test_grammar.pyc deleted file mode 100644 index bb1e6eae99868323e1f7ff2bafa8cb54f7ce210f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13010 zcmb_i34ENzb$_#3V_WjU;S;dE7KaVkvUVlo0~??C2u7GwMmC7m&aCuVtKHRpD3LGzSSJAqNQ|N1K-PO4=m9=4cwyv}wbyp@GExzc=6a?IHZLOe2=> zd)K^q@6F8n-pu&cmZ^>NU*EMXL-OZeK8MKS7pH!roFZk9B2Bb)o1H{j@w<_9jNem9 zH}G$ESB&g*GBMI|+S@>dqtOUBm=oxR80A^Xv80b87CcS_kBS6=LvUgwIFs}w);^o` zWPZ;fjSd?l`8gur6v@x6+YJa@&r$zh|XmlFc6RAH=xe25?^9#t% zqqu>EjpB+@%FbsET@xrDBelyeATvQ!#;EWe_?BR#36Yqir$_xHq@Sasews-i!_jIA zBh)Mc$3_At)eD>?0>?!HC)3Ckq-T&uh-Oh1Q9Fw4$yC}#T@%TkLZio#J(bKPG81XP z;$U~?Pb9kt&P*0(rV@Nnb}(FypN8$UxmkK^t za30#0I?t8~|9IwC`v)>itoYX|Vkgv#fef*Vmq{AwA$=n0d8FruUM8uuL%eJgFWXsF zt(VK|y*!f!?}<4tS4bCuqnMCjbmkEZyPEU@+KR!%JX=FL!SA)C7s^CEI|SB|ZWhY= z5ZFNaB%y2!flZ`O7Ru%j*h2ag*0+`PslwStdXWIzNiP=Q+z{A7`ZS@O7XlrmPZ!Gh zA!TO>TtNB^VO>aii2xUqUdmlvO1gy(5Op<@y^QoSX7aO%Oe2{oyzqF8Ftagn8%Y5- zO^kr5Y!~TPX@O^5iw0>iuc`&FaWgg4n2eO{CY=%`cpjmWwn)hy((U5d;t4X2z)fz z2T5-f_!zPek=`Wmv1A`6y;)%7ibqJFBk&BekCNUZ@JzCgk=`osEVAz-y-ncRWFMD& zznu=ioH=BlAcGu1m^n`}grzUj=qZtUk4ViW`+hRW&;jpH14PGZKQgI(T4c`^+2hH6 zfDE!VvO&Q9U#(AobbhC=(l93Y`hU#Z|>4=;zhE)iYLWWP$L zInwa800}j`Qo%zFU#}sBHoh9_y;Ss`MD`nGP6~)J^}bobLsj3ZA%&{GT~F1Wv>%0n zeU0=kUXI^owOF6Ku|)ro^kq!@9@CWlKEGf*dcfye*?$T|?LXHrcya%rh7t!Ujs)6u zGT9&2aP~-F0qaK<7R=4YxR-d>DHLZZe0A`>S?mz3717pp8rdI{?qVZNyiq~oPDP(`ivDn>2yHBRw=GXazhzd}>15xe5iIsM$(-)$ z%Qx8oFa{rP#1AcgNBT-`@fPVc@kj6svTu{_5qJsN-;+6mBv1T-%o2t;?C>nZt&|!? z)>O(YC9S#Q>dYf~#Z(*wt^@I{;nFD<)J+X)bSl{fWm@P4Iyl-R)DY=^jYNcEfpbZdy>+5?#3*qOYF&GI}%dV(J2aYwU!7e5;&DLKn4>!vZp|6 z1dhYDBCP={G!N1+Z!j`%iP9ww%~GDUg^BQUsS*cUB0`JGa~fWzY^%aF5}{U=KNhrP zL`b?;sfdtL`4y08iwJEhe<}#=5ush>XM(UiA}n_uovG|frG2()l^VfkcvU2^N)28) z71MXM8o{S`bwppS21l5Fma=P=E_2hf)rcf~9VQf*4q|DtEReI+;ImUZ?K(BmOj+X9 z0l0K5aVkJ5iNa_FQwCpw(0XMzC_N-{!#sVk8ZPTvN%lPMziSoQ4mEIQpM{u~7Ta?sZZeG1lU*IFUv&%o;n zaNH`6I9I6rG2nE|qVDEF+GVd)HZ2{dt2)FNsYg15ZrF+ZT9plYf)$>TW|^vHNZWdB zv#5I=G&E`-c5X&>>Z{3S6@I&HpVHTGv^-WPGt+Yuac@NEcZ8e}cpp<#c2HU0k%yJO zmKE${_7u^u&vCAn1}>Wm{c1ra7zfohUO4P$Fo8hp5PnEV=$RdL1Ie6aIdrJKBX-W-2H)V>v@zO2#^hoJQyGw z8F^Ug>skDffOs|&9}SRoj64={(4i0>SNgr&VEC=y5WMkf+8{A_PMQE`wKE-i2AkT{#3xBJ#Ep+DeSmxVq%zY;YL`zbGo3Zc zR_oK%$n(k^%^Z9kq`>`59sXHmQcpdE7hhKf2a*z9C`9!kF@Ck4%4)gL|HeBo-a+$Zh<-CD3tdfU zzokr^yjBk1Tomv2+sZ5>ZyApc&#J!^_SvLwrMVUAUnzYX7yR}+%A6WC{cAxmqeRi) z1T2`bkk1QW6TFiWjC?m>E@0$u8JW+}-vu&FLjQZgpii3ogJ3&x?zw}SMC13AzMW_5 z4|r;`u|H&>$k{*cJ})@K1Nlc|$!=2O>&pCpQN7ODAG2VuJN)>`*hT^8r(-!zU9VI3 zvryg7rCdb^|HgnTbMK%v+~qIEP+kuy@?GSr^A6|3ovy_%Lq0Rs#L;6+2dsZTH0xLI z%!(t1F?|>7*$Loo208)U!@!LI?sc91Cm-2OWrKe`)@h;t7fZ0fe=Geb0{lkl`vmx3 zrSIqA`JK`aNagn+g_YkrvTrH<5Sew9MYZ>E#8{6#l+urcj7KBJ25hL5evHg|G8=@A zweh})y^-u6lzv>gc_N~2BGfq-Eq)Rg4gO?2k}Zjy>5^|sexqQ?#KnbvllT2&9NLmh zY)ohR{ao7bGrp%l^Y#rVI8c;W zniP=r1LV-8lI<(nSGJwmzS5rpw6(o8`tzHZFp>-j{%OEF3W@WFO(~lzTy;*Vgzyyd ziM}D26L=9in(*gA8rFcGSi3f{DwS$$TajvQJ9Fjo_7y9ZuWVi6Cn2%)9T-cKiG^zu ztx?CJ0rK`|vsK9z@AN0pX&uSd{rShC$=H5YI#sC}^&ui{s+^DLJOG`>E9pew+;BQK zWc-;R%ZOvlcN8!cf_V5jF!=h1=%97~O{ZKuLf27_NBp(M>rQ;3;mH$ncH!k2s+uHi zi7mMTMiUx~1?bGi+gg(C*$HjQ_LkP{gw_@Sz|0x$wKT-qvJ=}lfGw?&X>Ey%$|Y75 z&czJQ=Ba`FW*C4Ie%ndc5XySKi0PyL48A*I4ZL;?PI-`eF&1MTjVVY z!Df$blr^Ud#?LP*Cz|~}IMd&pU_t^bffr;XQOjm|uk5EF(Rgv6Ni+`*d71v^#-38) zsyr6?NaNn2fuf%%U}iU#A+RWy-Z#1ks(V`+Wf|~N-s;Sib>btAIn(Xsi$i7cItz!Z zP%~02Ril_B0V>&PNuo7@wUd<9o*l|(w8>^*9ghaDN*u)VSv*Ib!`HExdGhcN6G$GU zY%&~K(RII^<=5$wQQ;L|$eV8D6Qn65obf16Wl*C>hCU#d19i&T2)_Z3@>K;%Vo?X< z>Y)uiOyC|)gbLg@T%%}DjINEzvw^O~$rdgl>iXmr7WE03{7HBqAr}inc`Zm}1uwsx z=a0|d_L^WmB$pm_eM>NT83(}}kC*9WO&lMV3zRy&HF0F4;ekIcy^r1Vj_+No127PIEGe?+P|7!8r?$Ko17nzr%GQ;>k$JoC;T+99Oj8CRAaf(N=d5$dZ?CR9V;a$xHb+O^6KHOY!5lVTiqh^(T zg7`|WuBWO9af9(JV3xzw8LnAp&y&T?U!C$XLwrPCB+&(KilU|bB3axH)^))e-yu>* z)KRL=mzT)m`mnB}|DP|dsajuNCj0W3F8&z4B+*gar6>7fze*O@jCHD6gz(c;SwXui4;wz56V5xK+gKqBmKi?`^rNQ-MAf=#bss zkyTW^DJjcJY0nRQ;L>UDkY7%4Ig4UYszijcPzu7uPWI*sJt+J*nxRp=N5gfSov!TB z${wTMjhj61S*fech{=TN`P=N5Sp5uTXDUN z2#jE{McB0^dZk2gLzhUOl}Iz^tV9o9shXt4Ln)10T3V#xl0;4ldN1o{J}-o;`gmg? z>xmZ-4ip^1)#62iLuur0nHm^7-d^MK%x>htI)HO3pywzI6~l_`7Cx9^I7SyI#c1>%Q8j_QiNuT;&49SF=du{aZ8lmCo`-+QB-9u$tZFly+G(NVdSjU z-Q8=-IIbG#?skf*hgT+e5NOGH)zWBu*c@Fo@jZ6Tn+Nm!xq$evjd$RQUTn+xoWeJQ zLnfawvUw|)MrGx6IKPKrphSxEujC2d>ZMDbU&#B@p{>Wur%R(ar1AR`g`zL%((m)K zWx+Tfq_uVjXw2qgoq-`_hhomlxYM(+)sQF(F+|j4$^{(Pani%cW6uDmnm0W9~GjB_Mg7%EBoEROC(#`i{K)JD9kSvU8R!GWcZ z!vdji%z(mKpE)>u#Se+WEiUFP=Erz4vNi%Lf(Z|418PR z#P?k)beVT^jlSP2926h9K90|3iQlGBon+oWD0{Ff^Wtbc%DgHzHuDB&XFP?`j+<7J zEPL$^zoN8un03cfVZfC8YWH>&voNDhF9*0bNARh@IQ7@=l2pvXwL1L`HTt=?bW(;h zof>a1Z7y#_PRUGX_fiqmHtxx}Y4K*Zh*vDH^=^Nt$W@k%;OTCC)YS=Y@r?SC5#1@R zq^j*wp1CB-v%r@y8l26^sb=Rw- zo{V7OAqsOc!WkLeCmc@Fv0?D3ykaW$B$| z1SqDUGx}X}`PQj7o1gdG3{Gzi@-9NGN%^ujw+S!e(#S^Ktp&OyGof s&iPv$!io3d(PCg756^(_IvObGp`7tWRhZ+%yG=np5zeM@PsMfr2epf{#{d8T diff --git a/test/simple_source/expression/06_huge_list.py b/test/simple_source/expression/06_huge_list.py index 6dcd97d0..c1d8b5f2 100644 --- a/test/simple_source/expression/06_huge_list.py +++ b/test/simple_source/expression/06_huge_list.py @@ -1,4 +1,8 @@ # We have more than 1<<16 elements +# In Python2 this causes an EXTENDED_ARG instruction to be emitted and then we can check +# handling that. + +# It also triggers the of special rules for expr32 and expr1024 [ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 545a0204..e226c607 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -44,6 +44,11 @@ class PythonParser(GenericASTBuilder): def __init__(self, SyntaxTree, start, debug): super(PythonParser, self).__init__(SyntaxTree, start, debug) # FIXME: customize per python parser version + + # These are the non-terminal we should collect into a list. + # For example instead of: + # stmts -> stmts stmt -> stmts stmt stmt ... + # collect as stmts -> stmt stmt ... nt_list = [ 'stmts', 'except_stmts', '_stmts', 'attributes', 'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_froms', diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 33985ac7..3aed264b 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2017 Rocky Bernstein +# Copyright (c) 2015-2018 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # Copyright (c) 1999 John Aycock @@ -294,8 +294,20 @@ class Python2Parser(PythonParser): # The order of opname listed is roughly sorted below if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'): + # We do this complicated test to speed up parsing of + # pathelogically long literals, especially those over 1024. + build_count = token.attr + thousands = (build_count//1024) + thirty32s = ((build_count//32) % 32) + if thirty32s > 0: + rule = "expr32 ::=%s" % (' expr' * 32) + self.add_unique_rule(rule, opname_base, build_count, customize) + if thousands > 0: + self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), + opname_base, build_count, customize) collection = opname_base[opname_base.find('_')+1:].lower() - rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname) + rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + + 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) self.add_unique_rules([ "expr ::= %s" % collection, rule], customize) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 941743ea..17b4cb8a 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -709,11 +709,24 @@ class Python3Parser(PythonParser): rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname)) self.add_unique_rule(rule, opname, token.attr, customize) if not is_LOAD_CLOSURE or v == 0: + build_count = token.attr + thousands = (build_count//1024) + thirty32s = ((build_count//32) % 32) + if thirty32s > 0: + rule = "expr32 ::=%s" % (' expr' * 32) + self.add_unique_rule(rule, opname_base, build_count, customize) + pass + if thousands > 0: + self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), + opname_base, build_count, customize) + pass collection = opname_base[opname_base.find('_')+1:].lower() - rule = '%s ::= %s%s' % (collection, 'expr ' * v, opname) + rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + + 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) self.add_unique_rules([ - 'expr ::= %s' % collection, + "expr ::= %s" % collection, rule], customize) + continue continue elif opname_base == 'BUILD_SLICE': if token.attr == 2: diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 77156433..8de9dd54 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -110,6 +110,17 @@ class Scanner2(Scanner): + @staticmethod + def extended_arg_val(arg): + """Return integer value of an EXTENDED_ARG operand. + In Python2 this always the operand value shifted 16 bits since + the operand is always 2 bytes. In Python 3.6+ this changes to one byte. + """ + if PYTHON3: + return (arg << 16) + else: + return (arg << long(16)) + @staticmethod def unmangle_name(name, classname): """Remove __ from the end of _name_ if it starts with __classname__ diff --git a/uncompyle6/semantics/helper.py b/uncompyle6/semantics/helper.py index f5e10374..19b89770 100644 --- a/uncompyle6/semantics/helper.py +++ b/uncompyle6/semantics/helper.py @@ -148,10 +148,12 @@ def flatten_list(node): for elem in node: if elem == 'expr1024': for subelem in elem: + assert subelem == 'expr32' for subsubelem in subelem: flat_elems.append(subsubelem) elif elem == 'expr32': for subelem in elem: + assert subelem == 'expr' flat_elems.append(subelem) else: flat_elems.append(elem)