From 4f0e5804388e4efe99da50fcbfb4b86ecb5312a8 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 10 Nov 2019 13:44:52 -0500 Subject: [PATCH 01/36] Update testenv pypy 3.6 --- test/test_pyenvlib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index e1ffecc6..c2b2706a 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -44,6 +44,7 @@ TEST_VERSIONS = ( "pypy3.5-5.9.0", "pypy3.5-6.0.0", "pypy3.6-7.1.0", + "pypy3.6-7.1.1", "native", ) + tuple(python_versions) From 82ea77c59226dee2af9ebc0098853d26e5e42272 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 10 Nov 2019 15:56:04 -0500 Subject: [PATCH 02/36] Python 3.0 bytecode decoding --- uncompyle6/parsers/parse3.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index c0e3a535..021e4125 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1437,8 +1437,18 @@ class Python3Parser(PythonParser): except_handler COME_FROM else_suitel opt_come_from_except """, - nop_func, + nop_func ) + if self.version == 3.0: + self.addRule( + """ + tryelsestmtl3 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK + except_handler COME_FROM else_suitel + come_froms + """, + nop_func + ) + custom_ops_processed.add(opname) elif opname_base in ("UNPACK_EX",): before_count, after_count = token.attr From 96dcdfd744f4064b5bf6292a669db6db5a2e14c4 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 10 Nov 2019 16:09:17 -0500 Subject: [PATCH 03/36] Last change but closer... --- uncompyle6/parsers/parse3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 021e4125..7a4a85b6 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1444,7 +1444,7 @@ class Python3Parser(PythonParser): """ tryelsestmtl3 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK except_handler COME_FROM else_suitel - come_froms + JUMP_FORWARD come_froms POP_TOP """, nop_func ) From 04c2240d630531eee72e46216f5d4d5df5f76c97 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 10 Nov 2019 17:23:33 -0500 Subject: [PATCH 04/36] Python 3.0 if/else handling --- test/bytecode_3.0/03_ifelse.pyc | Bin 0 -> 428 bytes test/simple_source/bug30/03_ifelse.py | 9 +++++++++ uncompyle6/parsers/parse3.py | 9 --------- uncompyle6/parsers/parse30.py | 3 +++ 4 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 test/bytecode_3.0/03_ifelse.pyc create mode 100644 test/simple_source/bug30/03_ifelse.py diff --git a/test/bytecode_3.0/03_ifelse.pyc b/test/bytecode_3.0/03_ifelse.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bfe7240fb97d22045118f622ed6597864473c3a6 GIT binary patch literal 428 zcmb79Jx>EM40Un^Aw(CJ0@ICIpp`m;5E3iu(hUh!*GqCrNiL`4JXB(Xl^?)A;%8xJ z;A13`pRMQTXXpB4di?SJ`P9O%C#b%o$WsCh@CNWmg@Cn^#Iy|jI9C++6#0SR1t2OA zlt_H8B|KLUXCOvu2^a Date: Sun, 10 Nov 2019 18:44:43 -0500 Subject: [PATCH 05/36] More Python 3.0 custom "if" statment handling. --- test/bytecode_3.0/03_ifelse.pyc | Bin 428 -> 692 bytes test/simple_source/bug30/03_ifelse.py | 11 +++++++++++ uncompyle6/parsers/parse30.py | 21 +++++++++++++++++++-- uncompyle6/semantics/customize3.py | 6 ++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/test/bytecode_3.0/03_ifelse.pyc b/test/bytecode_3.0/03_ifelse.pyc index bfe7240fb97d22045118f622ed6597864473c3a6..daa754a5a0a17d8186caf5281aa2fa55d21b8a5a 100644 GIT binary patch delta 304 zcmZ`zF>1p=5S%^9a%^Ns;l?Qg29-%20s$B4Ody@Xm_w4`5-b^=jx-7`V!yzBUXUh` zNBCKpb(JOyGtBMI&h1z9)4RV*N5j+SX_@0s8R`eZPe=u@2No29!eB30P(*mn;_usQ z**T#pk4aiRLiLRBQ&Jb9zJhT1Jx0g3K_}>991JEO!iiy2MCbgRye7V}#=Cv$VzRX)!SC{Pj3lgL+iHv2Q VX>!n8H`Z0*pT(n<$3&fj^cM&MG&cYM delta 82 zcmdnOx`x@%nunKbX77pEWCkR_2xL0|aWOBDNMT@TVPJ>?G8h?xHJCQqDlzi;X@HcJ RvH*$P#LUU*Ozt2i0{|H13>W|a diff --git a/test/simple_source/bug30/03_ifelse.py b/test/simple_source/bug30/03_ifelse.py index 97aea0d1..1af08601 100644 --- a/test/simple_source/bug30/03_ifelse.py +++ b/test/simple_source/bug30/03_ifelse.py @@ -7,3 +7,14 @@ def main(args, f): func(f, sys.stdout.buffer) else: func(sys.stdin.buffer, sys.stdout.buffer) + +# From Python 3.0 _markupbase.py. +# +# The Problem was in the way "if"s are generated in 3.0 which is sort +# of like a more optimized Python 2.6, with reduced extraneous jumps, +# but still 2.6-ish and not 2.7- or 3.1-ish. +def parse_marked_section(fn, i, rawdata, report=1): + if report: + j = 1 + fn(rawdata[i: j]) + return 10 diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 16f25e0f..c9288e5d 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -91,6 +91,9 @@ class Python30Parser(Python31Parser): except_suite ::= c_stmts POP_EXCEPT jump_except POP_TOP except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY _jump COME_FROM POP_TOP + + _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP + jump_except ::= JUMP_FORWARD COME_FROM POP_TOP jump_except ::= JUMP_ABSOLUTE COME_FROM POP_TOP or ::= expr jmp_false expr jmp_true expr @@ -101,8 +104,22 @@ class Python30Parser(Python31Parser): # JUMP_IF_FALSE # The below rules in fact are the same or similar. - jmp_true ::= JUMP_IF_TRUE POP_TOP - jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP + jmp_true ::= JUMP_IF_TRUE POP_TOP + jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP + jmp_false_then ::= JUMP_IF_FALSE POP_TOP + + # We don't have hacky THEN detection, so we do it + # in the grammar below which is also somewhat hacky. + + stmt ::= ifstmt30 + ifstmt30 ::= testexpr_then _ifstmts_jump30 + + testexpr_then ::= testfalse_then + testfalse_then ::= expr jmp_false_then + call_stmt ::= expr COME_FROM + _ifstmts_jump30 ::= c_stmts POP_TOP + + ################################################################################ for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index 9801538c..407b28e9 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -181,6 +181,12 @@ def customize_for_version3(self, version): # the iteration variable. These rules we can ignore # since we pick up the iteration variable some other way and # we definitely don't include in the source _[dd]. + TABLE_DIRECT.update({ + 'ifstmt30': ( '%|if %c:\n%+%c%-', + (0, "testexpr_then"), + (1, "_ifstmts_jump30") ), + }) + def n_comp_iter(node): if node[0] == "expr": n = node[0][0] From bdc24d7f514a2452dc554f35856ead7d34b0c012 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 11 Nov 2019 05:10:54 -0500 Subject: [PATCH 06/36] Add 3.0 gen_comp_body rule. --- test/bytecode_3.0/03_ifelse.pyc | Bin 692 -> 1149 bytes test/simple_source/bug30/03_ifelse.py | 10 ++++++++++ uncompyle6/parsers/parse30.py | 3 ++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/bytecode_3.0/03_ifelse.pyc b/test/bytecode_3.0/03_ifelse.pyc index daa754a5a0a17d8186caf5281aa2fa55d21b8a5a..b619b7198c5d273f08a5263973269a620f9026ca 100644 GIT binary patch delta 455 zcmYk0&q~8U5XQern%X~6JPM^23Q9l_u}8HCiqK0g1rI_AG1=N`n$%`jJXCt9s8`98 z2)=}m;{yoJrY&?}XXo4b=C|{0{goOg_GZOv{qzE?g~9bXpS1besM)W;je)R$u48?j znZ_*8eI5EXmnS{uLIP8QB+LzrEs!+Or4QiV#5>R@goL-;b2(=dF;X+!fH><|5nEiR zO-@sww}ThbTn0T9k+d1i>Y5%+gkXdob{$U0X-Y#5kCesNlOHLO8&rX1Dx!W?r-NCU z#p#`h#q)U5-qjsFRD`2ls={fQ4&zk%aUfKQWk-sMe;Wz4#llTwXEct)NW@Zv`IFb5 zBqMLQbS?H+{g`i($bpzFmLyd9lT|nSrTzK_Q$5d*L(i*mnHD*Lw#g#bF|!L}&sJ-e NY+ziK^nBU2tN&yeO*a4l delta 66 zcmey%v4vIMnunL`#Jm%+$qYz<3CMN;;$q2(^3u$~8Y~;#n3 Date: Mon, 11 Nov 2019 13:50:48 -0500 Subject: [PATCH 07/36] More Python 3.0 for JUMP elimination ... here, in except blocks. --- test/bytecode_3.0/02_try_except_except.pyc | Bin 409 -> 655 bytes .../simple_source/bug30/02_try_except_except.py | 10 ++++++++++ uncompyle6/parsers/parse30.py | 3 +++ 3 files changed, 13 insertions(+) diff --git a/test/bytecode_3.0/02_try_except_except.pyc b/test/bytecode_3.0/02_try_except_except.pyc index 2d6554652bdb39ff9572d2d1ba7743f176145139..a56e00218c251242dbbb5bb745eb9be5138518c6 100644 GIT binary patch delta 269 zcmYjMK??z45Pf5jK5RKSIJ#|aN-h$`MM}v5TMpPgY+_lv#X&Cal9M0dhxkFxyf2ZN z<~7Zm_vZU<#yL^1WwNSze@y21ix~SRYg-_AfCHQaHXFeSumw&G1IxHnp2we1xSL|N zw6ILrm9SS@asY-K1Mv`8DEKCXK?lyqoXd#76Evi%=V>Dz1uWe~IYQUn&-`iNbXK7o z1kpPDP;nrEr60K0jx>s%Ca3ezq+qKE5QC}2l%A!(ba~~ok$&QL^V*hvQZt$K8yB)G Ay8r+H delta 105 zcmeBYoylxs&BM#3a8x%snE?qf0oe{fTr2=2G8q_B7#La@8KQs;MuuPwrir#+75p?9 nfpVoF1;r(aMJ4fhspat{8AYjyDH=dAhKc8_Cs!~DvVhnCbq*6( diff --git a/test/simple_source/bug30/02_try_except_except.py b/test/simple_source/bug30/02_try_except_except.py index 56c729a9..3bbc94e7 100644 --- a/test/simple_source/bug30/02_try_except_except.py +++ b/test/simple_source/bug30/02_try_except_except.py @@ -7,3 +7,13 @@ def start_new_thread(function, args, kwargs={}): pass except: args() + +# Adapted from 3.0.1 code.py +# Bug is again JUMP_FORWARD elimination compared +# to earlier and later Pythons. +def interact(): + while 1: + try: + more = 1 + except KeyboardInterrupt: + more = 0 diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index f5e07b95..470fbe3d 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -121,6 +121,9 @@ class Python30Parser(Python31Parser): gen_comp_body ::= expr YIELD_VALUE COME_FROM POP_TOP + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts + COME_FROM POP_TOP END_FINALLY + ################################################################################ for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK From 4abdffecb95a28de1fcf8da4a9af2cd5d41af878 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 11 Nov 2019 19:07:58 -0500 Subject: [PATCH 08/36] More 3.0 control-flow rules... Much more is needed though --- test/bytecode_3.0/02_while1_if_while1.pyc | Bin 222 -> 484 bytes .../bug30/02_while1_if_while1.py | 5 +++ uncompyle6/parsers/parse30.py | 30 +++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/test/bytecode_3.0/02_while1_if_while1.pyc b/test/bytecode_3.0/02_while1_if_while1.pyc index c4282d8e527e49bcc16b56d23ff5e0643b3b91b9..7e44fe49d3bff37e971f19a5ef6ad92194147548 100644 GIT binary patch delta 282 zcmZ9F!3se^6o&sZ;}R~3wImx`3neU+y_AJgk_?61i)+f_V!;dC$MOkeTNmaqf-5H14aslFsLfNoC&1l6m?lagK?}OBnhr# z@GGDYSOG@iO7IKcnSCwuiLle4p#gsz43I{VdQ}~b%Q^d(iq8FfmNS|XByuRDlKKrj1-jp*~1H|{!V457rs5rTaF+hL^$Y*9` LX9QtZMqVZWoMj0* diff --git a/test/simple_source/bug30/02_while1_if_while1.py b/test/simple_source/bug30/02_while1_if_while1.py index 849bf110..f7ba63a5 100644 --- a/test/simple_source/bug30/02_while1_if_while1.py +++ b/test/simple_source/bug30/02_while1_if_while1.py @@ -7,3 +7,8 @@ while 1: raise RuntimeError else: raise RuntimeError + +# Adapted from 3.0.1 cgi.py +def _parseparam(s, end): + while end > 0 and s.count(''): + end = s.find(';') diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 470fbe3d..6fb0b545 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -135,11 +135,12 @@ class Python30Parser(Python31Parser): return_if_stmt ::= ret_expr RETURN_END_IF COME_FROM POP_TOP and ::= expr jmp_false expr come_from_opt whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt - JUMP_BACK COME_FROM POP_TOP POP_BLOCK COME_FROM_LOOP + JUMP_BACK come_froms POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP + # compare_chained is like x <= y <= z compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained1 _come_froms @@ -174,7 +175,34 @@ class Python30Parser(Python31Parser): compare_chained2 COME_FROM """) + self.check_reduce['iflaststmtl'] = 'AST' + # self.check_reduce['ifelsestmt'] = 'AST' return + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + invalid = super(Python30Parser, + self).reduce_is_invalid(rule, ast, + tokens, first, last) + if invalid: + return invalid + if ( + rule[0] in ("iflaststmtl",) and ast[0] == "testexpr" + ): + testexpr = ast[0] + if testexpr[0] == "testfalse": + testfalse = testexpr[0] + if testfalse[1] == "jmp_false": + jmp_false = testfalse[1] + if last == len(tokens): + last -= 1 + while (isinstance(tokens[first].offset, str) and first < last): + first += 1 + if first == last: + return True + while (first < last and isinstance(tokens[last].offset, str)): + last -= 1 + return not (tokens[first].offset <= jmp_false[0].attr <= tokens[last].offset) + pass class Python30ParserSingle(Python30Parser, PythonParserSingle): From 9f250b49eeff4cb13b571a513950b4024309f6a4 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 11 Nov 2019 19:58:35 -0500 Subject: [PATCH 09/36] Cope more JUMP/POP_IF not being in 3.0... more is probably needed though. --- test/bytecode_3.0_run/04_and_del.pyc | Bin 801 -> 1110 bytes test/simple_source/bug30/04_and_del.py | 10 ++++++++++ uncompyle6/parsers/parse30.py | 19 +++++++++++++++---- uncompyle6/semantics/customize3.py | 6 +++++- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/bytecode_3.0_run/04_and_del.pyc b/test/bytecode_3.0_run/04_and_del.pyc index 06a2ddebf48c09cbb89cf15b4ff01e7364b40ff4..6d5353397255ac99ed46070c2864a6ba8ceba71a 100644 GIT binary patch delta 357 zcmZ3;c8x>9nunLGllfF^G6ND|2eKW2xR{f1qQa)=76yhWCWcfdh7?XP;$moKW+-51 zU@T!|Xkuhw1j%qgC4n+Lm{MR_R81)y48a=Q8?RnwtWRc!+RXx_8G+atWVZy6s9|8p z2ijA^$WXunq?j0LnHg%Bff_-o{4{_Bg9b=bDI1VT%P-1JEGY%?GfHxEz>JdAijq=J zxMWFc5kwPEgrQUsNEB!0PPSl@mjM}&k(if~lM1r87-R?oqX?s)TZjhh Date: Tue, 12 Nov 2019 06:08:30 -0500 Subject: [PATCH 10/36] More 3.0 control flow pattern fixups --- test/bytecode_3.0/03_ifelse.pyc | Bin 1149 -> 1386 bytes test/simple_source/bug30/03_ifelse.py | 11 +++++++++-- uncompyle6/parsers/parse30.py | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/bytecode_3.0/03_ifelse.pyc b/test/bytecode_3.0/03_ifelse.pyc index b619b7198c5d273f08a5263973269a620f9026ca..fa86044c03c436fc60a1020c1bcd65c16795a348 100644 GIT binary patch delta 497 zcmZuuy-EW?5T3canB+VKMG!3vk$@Y5g|V~|grH5D>jXJ2n~h04E+Knc1R@q%*q-+e zV((iBzKGx>D9$VfEF74bZ~pe1-TkV3__dQ@&EI=^yOQun9Kxrhx|P=&I}ZStF7!(f ziWN#qj|_{KhoZG`V1cl3VGJ-vxOV{4T|iD#&f(m$m_N5Ym*(=%dV|j}Y~tCqHb`J# z!+}Yq;U+!TuXSwVF$XN{m#W(oWMG*c=A4!QGlUQyX$z;gb>26**Z5>cdX(xcPGvE2 zMZ>0HFlzLG;F kZ=#4-(EJgAWn9Ou@QQEuXv_T5TpW7mKDSnbV6|HQ0ggmii2wiq delta 282 zcmaFG^_N55nunJw-uPr}G6ND|0{!HgXYKuagLF`bnM z*;C2|Br=OL^NLFn^O93bIe=WxlGLKaq@2mA%o>bLllz#(CU0a8X7 Date: Tue, 12 Nov 2019 06:37:43 -0500 Subject: [PATCH 11/36] 3.0 tweak in iflastsmtl reduction validation --- uncompyle6/parsers/parse30.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 00c79605..779768fe 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -212,7 +212,10 @@ class Python30Parser(Python31Parser): return True while (first < last and isinstance(tokens[last].offset, str)): last -= 1 - return not (tokens[first].offset <= jmp_false[0].attr <= tokens[last].offset) + if rule[0] == "iflaststmtl": + return not (jmp_false[0].attr <= tokens[last].offset) + else: + return not (tokens[first].offset <= jmp_false[0].attr <= tokens[last].offset) pass From 659f37585b3a9099083098ab82e71a88eac07d96 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 12 Nov 2019 16:31:43 -0500 Subject: [PATCH 12/36] Bug in 3.0 rule in "jump_absolute_else" ... and disallowing "else" to the wrong place. --- test/bytecode_3.0/03_ifelse.pyc | Bin 1386 -> 1641 bytes test/simple_source/bug30/03_ifelse.py | 13 +++++++++++++ uncompyle6/parsers/parse30.py | 15 ++++++++++++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/test/bytecode_3.0/03_ifelse.pyc b/test/bytecode_3.0/03_ifelse.pyc index fa86044c03c436fc60a1020c1bcd65c16795a348..aa09846c53bed72615dd511444fccf712df43377 100644 GIT binary patch delta 284 zcmaFG^^!;4nunK*PvvxMG6ND|0}*149%GLkb&1um<}^|3;SjWM-&Z z79b5`JA>5%i3(kY8U_ZJSXG8vW`-I@hI|PI#u_Gud>$YLK_cuNCl3!dCp!RAy%0M9 diff --git a/test/simple_source/bug30/03_ifelse.py b/test/simple_source/bug30/03_ifelse.py index fcb813fd..a5b9d655 100644 --- a/test/simple_source/bug30/03_ifelse.py +++ b/test/simple_source/bug30/03_ifelse.py @@ -35,3 +35,16 @@ def __instancecheck__(subtype, subclass, cls): if subtype: if (cls and subclass): return False + + +# Adapted from 3.0.1 abc.py +# Bug was rule in "jump_absolute_else" and disasllowing +# "else" to the wrong place. + +def _strptime(locale_time, found_zone, time): + for tz_values in locale_time: + if found_zone: + if (time and found_zone): + break + else: + break diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 779768fe..7b4023d6 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -43,7 +43,11 @@ class Python30Parser(Python31Parser): else_suitel ::= l_stmts COME_FROM_LOOP JUMP_BACK + jump_absolute_else ::= COME_FROM JUMP_ABSOLUTE COME_FROM POP_TOP + + ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel + iflaststmtl ::= testexpr c_stmts_opt jb_pop_top iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP @@ -187,7 +191,7 @@ class Python30Parser(Python31Parser): self.check_reduce["iflaststmtl"] = "AST" self.check_reduce['ifstmt'] = "AST" - # self.check_reduce["ifelsestmt"] = "AST" + self.check_reduce["ifelsestmtc"] = "AST" return def reduce_is_invalid(self, rule, ast, tokens, first, last): @@ -197,12 +201,17 @@ class Python30Parser(Python31Parser): if invalid: return invalid if ( - rule[0] in ("iflaststmtl", "ifstmt") and ast[0] == "testexpr" + rule[0] in ("iflaststmtl", "ifstmt", "ifelsestmtc") and ast[0] == "testexpr" ): testexpr = ast[0] if testexpr[0] == "testfalse": testfalse = testexpr[0] - if testfalse[1] == "jmp_false": + if rule[0] == "ifelsestmtc" and ast[2] == "jump_absolute_else": + jump_absolute_else = ast[2] + come_from = jump_absolute_else[2] + return come_from == "COME_FROM" and come_from.attr < tokens[first].offset + pass + elif testfalse[1] == "jmp_false": jmp_false = testfalse[1] if last == len(tokens): last -= 1 From 065fd13b81496a0cbbd89f7067a2cc4bb47c48bb Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 12 Nov 2019 17:04:35 -0500 Subject: [PATCH 13/36] Small tweaks --- test/simple_source/bug30/03_ifelse.py | 2 +- uncompyle6/parsers/parse30.py | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/test/simple_source/bug30/03_ifelse.py b/test/simple_source/bug30/03_ifelse.py index a5b9d655..0f36b953 100644 --- a/test/simple_source/bug30/03_ifelse.py +++ b/test/simple_source/bug30/03_ifelse.py @@ -38,7 +38,7 @@ def __instancecheck__(subtype, subclass, cls): # Adapted from 3.0.1 abc.py -# Bug was rule in "jump_absolute_else" and disasllowing +# Bug was rule in "jump_absolute_else" and disallowing # "else" to the wrong place. def _strptime(locale_time, found_zone, time): diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 7b4023d6..6d83dbf2 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -141,7 +141,9 @@ class Python30Parser(Python31Parser): POP_TOP END_FINALLY return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP + and ::= expr jmp_false expr come_from_opt + whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt JUMP_BACK come_froms POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns @@ -192,6 +194,8 @@ class Python30Parser(Python31Parser): self.check_reduce["iflaststmtl"] = "AST" self.check_reduce['ifstmt'] = "AST" self.check_reduce["ifelsestmtc"] = "AST" + self.check_reduce["ifelsestmt"] = "AST" + # self.check_reduce["and"] = "AST" return def reduce_is_invalid(self, rule, ast, tokens, first, last): @@ -200,17 +204,25 @@ class Python30Parser(Python31Parser): tokens, first, last) if invalid: return invalid + lhs = rule[0] if ( - rule[0] in ("iflaststmtl", "ifstmt", "ifelsestmtc") and ast[0] == "testexpr" + lhs in ("iflaststmtl", "ifstmt", + "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr" ): testexpr = ast[0] if testexpr[0] == "testfalse": testfalse = testexpr[0] - if rule[0] == "ifelsestmtc" and ast[2] == "jump_absolute_else": + if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else": jump_absolute_else = ast[2] come_from = jump_absolute_else[2] return come_from == "COME_FROM" and come_from.attr < tokens[first].offset pass + elif lhs == "ifelsestmt" and ast[2] == "jf_cf_pop": + come_froms = ast[2][1] + for come_from in come_froms: + if come_from.attr < tokens[first].offset: + return True + return False elif testfalse[1] == "jmp_false": jmp_false = testfalse[1] if last == len(tokens): @@ -225,6 +237,14 @@ class Python30Parser(Python31Parser): return not (jmp_false[0].attr <= tokens[last].offset) else: return not (tokens[first].offset <= jmp_false[0].attr <= tokens[last].offset) + pass + pass + pass + # elif lhs == "and": + # jmp_false = ast[1] + # if jmp_false == "jmp_false": + # return jmp_false[0].attr > tokens[last].offset + pass From d852f23962ff9d03bd74be43adac84dfa8c0ec62 Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 12 Nov 2019 17:48:16 -0500 Subject: [PATCH 14/36] 3.3 "yield from" semantic action fix --- uncompyle6/semantics/customize3.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index 67edbadb..680ddf60 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -207,11 +207,14 @@ def customize_for_version3(self, version): # FIXME: perhaps this can be folded into the 3.4+ case? def n_yield_from(node): assert node[0] == "expr" - assert node[0][0] == "get_iter" - # Skip over yield_from.expr.get_iter which adds an - # extra iter(). Maybe we can do in tranformation phase instead? - template = ("yield from %c", (0, "expr")) - self.template_engine(template, node[0][0]) + if node[0][0] == "get_iter": + # Skip over yield_from.expr.get_iter which adds an + # extra iter(). Maybe we can do in tranformation phase instead? + template = ("yield from %c", (0, "expr")) + self.template_engine(template, node[0][0]) + else: + template = ("yield from %c", (0, "attribute")) + self.template_engine(template, node[0][0][0]) self.prune() self.n_yield_from = n_yield_from From cda0154594d5ead8da2b8c1da9e073ff59b6a59d Mon Sep 17 00:00:00 2001 From: rocky Date: Tue, 12 Nov 2019 23:33:36 -0500 Subject: [PATCH 15/36] Pypy 3.6.9 tolerance --- Makefile | 6 +++++- test/Makefile | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 65665b94..9a0dd4cc 100644 --- a/Makefile +++ b/Makefile @@ -47,10 +47,14 @@ check-3.8: # Skip for now 2.6 5.0 5.3 5.6 5.8: -#:PyPy pypy3-2.4.0 Python 3: +#:PyPy pypy3-2.4.0 Python 3.6.1: 7.1 pypy-3.2 2.4: $(MAKE) -C test $@ +#:PyPy pypy3-2.4.0 Python 3.6.9: +7.2: + $(MAKE) -C test $@ + #: Run py.test tests pytest: $(MAKE) -C pytest check diff --git a/test/Makefile b/test/Makefile index 30f1ccb2..598154cb 100644 --- a/test/Makefile +++ b/test/Makefile @@ -332,12 +332,18 @@ pypy-2.7 5.0 5.3 6.0: pypy-3.2 2.4: $(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify -#: PyPy 5.0.x with Python 3.6 ... +#: PyPy 5.0.x with Python 3.6.1 ... check-bytecode-pypy3.6: 7.1 7.1: $(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify +#: PyPy 5.0.x with Python 3.6.9 +check-bytecode-pypy3.6: 7.2 +7.2: + $(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run + $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify + clean: clean-py-dis clean-dis clean-unverified From 78a595c8cf13480abe5d874e542834d3f4eba563 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 13 Nov 2019 20:35:44 -0500 Subject: [PATCH 16/36] Bang on 3.0.1 control flow... more word is needed though --- uncompyle6/parsers/parse30.py | 34 +++++++++++++++++++++++----------- uncompyle6/parsers/treenode.py | 8 ++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 6d83dbf2..29f25454 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -45,14 +45,16 @@ class Python30Parser(Python31Parser): jump_absolute_else ::= COME_FROM JUMP_ABSOLUTE COME_FROM POP_TOP + jump_cf_pop ::= _come_froms _jump _come_froms POP_TOP + + ifelsestmt ::= testexpr c_stmts_opt jump_cf_pop else_suite COME_FROM + ifelsestmtl ::= testexpr c_stmts_opt jump_cf_pop else_suitel ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec - ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel + ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec iflaststmtl ::= testexpr c_stmts_opt jb_pop_top iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP - jf_cf_pop ::= JUMP_FORWARD come_froms POP_TOP - ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite COME_FROM withasstmt ::= expr setupwithas store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_FINALLY @@ -142,7 +144,7 @@ class Python30Parser(Python31Parser): return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP - and ::= expr jmp_false expr come_from_opt + and ::= expr jmp_false_then expr come_from_opt whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt JUMP_BACK come_froms POP_TOP POP_BLOCK COME_FROM_LOOP @@ -166,6 +168,7 @@ class Python30Parser(Python31Parser): ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_froms + jump_forward_else ::= JUMP_FORWARD ELSE jump_absolute_else ::= JUMP_ABSOLUTE ELSE whilestmt ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK @@ -189,13 +192,14 @@ class Python30Parser(Python31Parser): compare_chained2 COME_FROM ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM + and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM """) self.check_reduce["iflaststmtl"] = "AST" self.check_reduce['ifstmt'] = "AST" self.check_reduce["ifelsestmtc"] = "AST" self.check_reduce["ifelsestmt"] = "AST" - # self.check_reduce["and"] = "AST" + # self.check_reduce["and"] = "stmt" return def reduce_is_invalid(self, rule, ast, tokens, first, last): @@ -217,11 +221,22 @@ class Python30Parser(Python31Parser): come_from = jump_absolute_else[2] return come_from == "COME_FROM" and come_from.attr < tokens[first].offset pass - elif lhs == "ifelsestmt" and ast[2] == "jf_cf_pop": - come_froms = ast[2][1] + elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop": + jump_cf_pop = ast[2] + come_froms = jump_cf_pop[0] for come_from in come_froms: if come_from.attr < tokens[first].offset: return True + come_froms = jump_cf_pop[2] + if come_froms == "COME_FROM": + if come_froms.attr < tokens[first].offset: + return True + pass + elif come_froms == "_come_froms": + for come_from in come_froms: + if come_from.attr < tokens[first].offset: + return True + return False elif testfalse[1] == "jmp_false": jmp_false = testfalse[1] @@ -241,10 +256,7 @@ class Python30Parser(Python31Parser): pass pass # elif lhs == "and": - # jmp_false = ast[1] - # if jmp_false == "jmp_false": - # return jmp_false[0].attr > tokens[last].offset - + # return tokens[last+1] == "JUMP_FORWARD" pass diff --git a/uncompyle6/parsers/treenode.py b/uncompyle6/parsers/treenode.py index 9f468072..32132893 100644 --- a/uncompyle6/parsers/treenode.py +++ b/uncompyle6/parsers/treenode.py @@ -54,3 +54,11 @@ class SyntaxTree(spark_AST): rv += "\n" + child i += 1 return rv + + def first_child(self): + if len(self) > 0: + child = self[0] + if not isinstance(child, SyntaxTree): + return child + return self[0].first_child() + return self From 7c8f3cc9ec129bc6fe80129705467580f05f5bab Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 14 Nov 2019 03:57:14 -0500 Subject: [PATCH 17/36] Two 3.0 rules ... - ifstmtlastl - ifnotstmt30 --- uncompyle6/parsers/parse30.py | 9 +++++++-- uncompyle6/semantics/customize3.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 29f25454..81739374 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -53,6 +53,8 @@ class Python30Parser(Python31Parser): ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec iflaststmtl ::= testexpr c_stmts_opt jb_pop_top + iflaststmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK COME_FROM POP_TOP + iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP @@ -111,6 +113,7 @@ class Python30Parser(Python31Parser): # The below rules in fact are the same or similar. jmp_true ::= JUMP_IF_TRUE POP_TOP + jmp_true_then ::= JUMP_IF_TRUE _come_froms POP_TOP jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP jmp_false_then ::= JUMP_IF_FALSE POP_TOP @@ -118,10 +121,12 @@ class Python30Parser(Python31Parser): # in the grammar below which is also somewhat hacky. stmt ::= ifstmt30 - ifstmt30 ::= testexpr_then _ifstmts_jump30 + stmt ::= ifnotstmt30 + ifstmt30 ::= testfalse_then _ifstmts_jump30 + ifnotstmt30 ::= testtrue_then _ifstmts_jump30 - testexpr_then ::= testfalse_then testfalse_then ::= expr jmp_false_then + testtrue_then ::= expr jmp_true_then call_stmt ::= expr COME_FROM _ifstmts_jump30 ::= c_stmts POP_TOP diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index 680ddf60..fd6af80b 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -183,7 +183,10 @@ def customize_for_version3(self, version): # we definitely don't include in the source _[dd]. TABLE_DIRECT.update({ 'ifstmt30': ( "%|if %c:\n%+%c%-", - (0, "testexpr_then"), + (0, "testfalse_then"), + (1, "_ifstmts_jump30") ), + 'ifnotstmt30': ( "%|if not %c:\n%+%c%-", + (0, "testtrue_then"), (1, "_ifstmts_jump30") ), "or30": ( "%c or %c", (0, "expr"), From fff6f82dd7a1103ce484128d7c66eea5eb9cbb60 Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 14 Nov 2019 10:11:28 -0500 Subject: [PATCH 18/36] expand 3.0 jump_except rule --- test/test_pyenvlib.py | 1 + uncompyle6/parsers/parse30.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index c2b2706a..37df5eb3 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -45,6 +45,7 @@ TEST_VERSIONS = ( "pypy3.5-6.0.0", "pypy3.6-7.1.0", "pypy3.6-7.1.1", + "pypy3.6-7.2.0", "native", ) + tuple(python_versions) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 81739374..ce26b596 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -100,10 +100,12 @@ class Python30Parser(Python31Parser): except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY _jump COME_FROM POP_TOP + except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY + _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP - jump_except ::= JUMP_FORWARD COME_FROM POP_TOP - jump_except ::= JUMP_ABSOLUTE COME_FROM POP_TOP + jump_except ::= _jump COME_FROM POP_TOP + or ::= expr jmp_false expr jmp_true expr or ::= expr jmp_true expr From 2e36551c026053b2ff7ce1794ce905e58edbd48b Mon Sep 17 00:00:00 2001 From: rocky Date: Thu, 14 Nov 2019 18:31:46 -0500 Subject: [PATCH 19/36] Remove more 3.0 parse errors... However this doesn't mean that some wrong rules still don't kid in. We still have control-flow "if/and/else" vs "if/if/else" problems. --- uncompyle6/parsers/parse30.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index ce26b596..0cf077b4 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -258,7 +258,11 @@ class Python30Parser(Python31Parser): if rule[0] == "iflaststmtl": return not (jmp_false[0].attr <= tokens[last].offset) else: - return not (tokens[first].offset <= jmp_false[0].attr <= tokens[last].offset) + jmp_false_target = jmp_false[0].attr + if tokens[first].offset > jmp_false_target: + return True + return ( + (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD") pass pass pass From 1b4335edf1ed224fde55e2affc6742c0aea66e27 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 07:48:23 -0500 Subject: [PATCH 20/36] Add 3.0.1 while loop rule --- uncompyle6/parsers/parse30.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 0cf077b4..5d6498dc 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -157,7 +157,8 @@ class Python30Parser(Python31Parser): JUMP_BACK come_froms POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP - + whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt + come_froms POP_TOP POP_BLOCK COME_FROM_LOOP # compare_chained is like x <= y <= z From 0441fbc6167a7bc97d4e702912710e75e0316341 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 09:12:53 -0500 Subject: [PATCH 21/36] 3.0.1 "ret_or", "ret_and", and "or" rules --- uncompyle6/parsers/parse30.py | 11 +++++++---- uncompyle6/semantics/customize3.py | 4 ---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 5d6498dc..8a32fc91 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -137,9 +137,11 @@ class Python30Parser(Python31Parser): except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts COME_FROM POP_TOP END_FINALLY - expr ::= or30 - ret_or ::= or30 - or30 ::= expr JUMP_IF_TRUE COME_FROM POP_TOP expr come_from_opt + jit_cf_pop ::= JUMP_IF_TRUE _come_froms POP_TOP + jif_cf_pop ::= JUMP_IF_FASLE _come_froms POP_TOP + or ::= expr jit_cf_pop expr come_from_opt + ret_or ::= expr jit_cf_pop expr come_from_opt + ret_and ::= expr jif_cf_pop expr come_from_opt ################################################################################ for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK @@ -198,7 +200,8 @@ class Python30Parser(Python31Parser): compare_chained1 COME_FROM compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained2 COME_FROM - ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM + ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM + ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM """) diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index fd6af80b..e52f0319 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -188,10 +188,6 @@ def customize_for_version3(self, version): 'ifnotstmt30': ( "%|if not %c:\n%+%c%-", (0, "testtrue_then"), (1, "_ifstmts_jump30") ), - "or30": ( "%c or %c", - (0, "expr"), - (4, "expr") ), - }) def n_comp_iter(node): From c8252ca9cb88f072ecca878111940fece66f8941 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 09:27:47 -0500 Subject: [PATCH 22/36] 3.0 import_from rule --- uncompyle6/parsers/parse30.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 8a32fc91..4f11eeb8 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -109,6 +109,8 @@ class Python30Parser(Python31Parser): or ::= expr jmp_false expr jmp_true expr or ::= expr jmp_true expr + import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist _come_froms POP_TOP + ################################################################################ # In many ways 3.0 is like 2.6. One similarity is there is no JUMP_IF_TRUE and # JUMP_IF_FALSE From 49de5b5c9d5dd7dc44a6e1398b24b953e6c7bc53 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 14:07:37 -0500 Subject: [PATCH 23/36] add 3.0 iflaststmt rule --- uncompyle6/parsers/parse30.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 4f11eeb8..b2e9e30a 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -52,6 +52,7 @@ class Python30Parser(Python31Parser): ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec + iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM iflaststmtl ::= testexpr c_stmts_opt jb_pop_top iflaststmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK COME_FROM POP_TOP From af2ed318717ec8004acd0fe9f409fea74c84614d Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 15:14:33 -0500 Subject: [PATCH 24/36] Add 3.0 whilestmt rule --- uncompyle6/parsers/parse30.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index b2e9e30a..512827ef 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -159,7 +159,7 @@ class Python30Parser(Python31Parser): and ::= expr jmp_false_then expr come_from_opt whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt - JUMP_BACK come_froms POP_TOP POP_BLOCK COME_FROM_LOOP + JUMP_BACK _come_froms POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt From 47c847644efe4fa230ce3346a1461524167cb2c2 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 19:23:08 -0500 Subject: [PATCH 25/36] Modify 3.0 _ifstmts_jump rule --- uncompyle6/parsers/parse30.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 512827ef..d36e6dfd 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -104,6 +104,7 @@ class Python30Parser(Python31Parser): except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP + _ifstmts_jump ::= c_stmts_opt come_froms POP_TOP JUMP_FORWARD _come_froms jump_except ::= _jump COME_FROM POP_TOP From 4e9d8783d16d0845022e79192be1cc3a373f7759 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 21:32:31 -0500 Subject: [PATCH 26/36] Add Python 3.0.1 list_comp rule --- uncompyle6/parsers/parse26.py | 2 +- uncompyle6/parsers/parse30.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index 33012d0d..9b7c35d4 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -254,7 +254,7 @@ class Python26Parser(Python2Parser): POP_TOP jb_pb_come_from generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM - list_if ::= list_if ::= expr jmp_false_then list_iter + list_if ::= expr jmp_false_then list_iter ''' def p_ret26(self, args): diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index d36e6dfd..7681bd5e 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -75,6 +75,9 @@ class Python30Parser(Python31Parser): list_comp ::= list_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK + list_comp ::= list_comp_header + LOAD_FAST FOR_ITER store comp_iter + JUMP_BACK _come_froms POP_TOP JUMP_BACK set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST set_comp ::= set_comp_header @@ -86,6 +89,16 @@ class Python30Parser(Python31Parser): LOAD_FAST FOR_ITER store dict_comp_iter JUMP_BACK + # From Python 2.6 + + + list_iter ::= list_if JUMP_BACK + list_iter ::= list_if JUMP_BACK _come_froms POP_TOP + lc_body ::= LOAD_NAME expr LIST_APPEND + lc_body ::= LOAD_FAST expr LIST_APPEND + list_if ::= expr jmp_false_then list_iter + ############# + dict_comp_iter ::= expr expr ROT_TWO expr STORE_SUBSCR # JUMP_IF_TRUE POP_TOP as a replacement From bc8907e752e0f512843e976d48360ecea7fa3cba Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 22:19:17 -0500 Subject: [PATCH 27/36] Guard again improper assert transform... we see this happen in getheader() from 3.0.1/lib/python3.0/http/client.pyc --- uncompyle6/semantics/transform.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index 7156db40..f80fe23d 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -115,11 +115,14 @@ class TreeTransform(GenericASTTraversal, object): call = expr[0] LOAD_ASSERT = call[0] - expr = call[1][0] - node = SyntaxTree( - kind, - [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1] - ) + if isinstance(call[1], SyntaxTree): + expr = call[1][0] + node = SyntaxTree( + kind, + [assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1] + ) + pass + pass else: # ifstmt # 0. testexpr (2) From 44f0ba0efb564cdc1c7b500691f9214211754db5 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 15 Nov 2019 23:39:36 -0500 Subject: [PATCH 28/36] Add 3.0 try/except rule --- uncompyle6/parsers/parse30.py | 7 +++++++ uncompyle6/semantics/customize3.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 7681bd5e..5199813e 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -12,6 +12,8 @@ class Python30Parser(Python31Parser): def p_30(self, args): """ + pt_bp ::= POP_TOP POP_BLOCK + assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA @@ -89,6 +91,11 @@ class Python30Parser(Python31Parser): LOAD_FAST FOR_ITER store dict_comp_iter JUMP_BACK + stmt ::= try_except30 + try_except30 ::= SETUP_EXCEPT suite_stmts_opt + _come_froms pt_bp + except_handler opt_come_from_except + # From Python 2.6 diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index e52f0319..9875126d 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -182,12 +182,16 @@ def customize_for_version3(self, version): # since we pick up the iteration variable some other way and # we definitely don't include in the source _[dd]. TABLE_DIRECT.update({ - 'ifstmt30': ( "%|if %c:\n%+%c%-", + "ifstmt30": ( "%|if %c:\n%+%c%-", (0, "testfalse_then"), (1, "_ifstmts_jump30") ), - 'ifnotstmt30': ( "%|if not %c:\n%+%c%-", - (0, "testtrue_then"), - (1, "_ifstmts_jump30") ), + "ifnotstmt30": ( "%|if not %c:\n%+%c%-", + (0, "testtrue_then"), + (1, "_ifstmts_jump30") ), + "try_except30": ( "%|try:\n%+%c%-%c\n\n", + (1, "suite_stmts_opt"), + (4, "except_handler") ), + }) def n_comp_iter(node): From 6a81a752a700b3ef52b3e8068826e61b678ee151 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 00:59:58 -0500 Subject: [PATCH 29/36] Adjust 3.0 iflastsmtl rule --- uncompyle6/parsers/parse30.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 5199813e..a6e13c00 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -56,7 +56,7 @@ class Python30Parser(Python31Parser): iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM iflaststmtl ::= testexpr c_stmts_opt jb_pop_top - iflaststmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK COME_FROM POP_TOP + iflaststmtl ::= testexpr c_stmts_opt come_froms JUMP_BACK COME_FROM POP_TOP iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP @@ -90,6 +90,9 @@ class Python30Parser(Python31Parser): dict_comp ::= dict_comp_header LOAD_FAST FOR_ITER store dict_comp_iter JUMP_BACK + dict_comp ::= dict_comp_header + LOAD_FAST FOR_ITER store dict_comp_iter + JUMP_BACK _come_froms POP_TOP JUMP_BACK stmt ::= try_except30 try_except30 ::= SETUP_EXCEPT suite_stmts_opt From 047e27c96629ddd0212535b46a7d72a1de4c3fdb Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 09:10:14 -0500 Subject: [PATCH 30/36] 3.0 assert2... Not like other 3.x due to the lack of POP_JUMP_IF --- uncompyle6/parsers/parse30.py | 5 +++++ uncompyle6/scanners/scanner3.py | 14 +++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index a6e13c00..7baef5a3 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -15,6 +15,9 @@ class Python30Parser(Python31Parser): pt_bp ::= POP_TOP POP_BLOCK assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP + assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 + come_froms + return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA @@ -212,7 +215,9 @@ class Python30Parser(Python31Parser): COME_FROM_LOOP whilestmt ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM_LOOP + assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 + return_if_lambda ::= RETURN_END_IF_LAMBDA except_suite ::= c_stmts POP_EXCEPT jump_except whileelsestmt ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 6a025486..661a5f17 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -257,21 +257,29 @@ class Scanner3(Scanner): # RAISE_VARARGS then we have a "raise" statement # else we have an "assert" statement. if self.version == 3.0: - # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here + # Like 2.6, 3.0 doesn't have POP_JUMP_IF... so we have + # to go through more machinations assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n + if assert_can_follow: + prev_inst = self.insts[i - 1] + assert_can_follow = ( + prev_inst.opname in ("JUMP_IF_TRUE", "JUMP_IF_FALSE") + and i + 1 < n ) + jump_if_inst = prev_inst else: assert_can_follow = ( inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE") and i + 1 < n ) + jump_if_inst = inst if assert_can_follow: next_inst = self.insts[i + 1] if ( next_inst.opname == "LOAD_GLOBAL" and next_inst.argval == "AssertionError" - and inst.argval + and jump_if_inst.argval ): - raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] + raise_idx = self.offset2inst_index[self.prev_op[jump_if_inst.argval]] raise_inst = self.insts[raise_idx] if raise_inst.opname.startswith("RAISE_VARARGS"): self.load_asserts.add(next_inst.offset) From dbf2729f76459201aa0d6f7fa258dd2607c18770 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 12:23:09 -0500 Subject: [PATCH 31/36] expand 3.0 "continue" detection --- uncompyle6/scanners/scanner3.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 661a5f17..7cf2030c 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -482,6 +482,12 @@ class Scanner3(Scanner): and self.insts[i + 1].opname == "JUMP_FORWARD" ) + if (self.version == 3.0 and self.insts[i + 1].opname == "JUMP_FORWARD" + and not is_continue): + target_prev = self.offset2inst_index[self.prev_op[target]] + is_continue = ( + self.insts[target_prev].opname == "SETUP_LOOP") + if is_continue or ( inst.offset in self.stmts and ( From d21d93fd84f66a55be3938f551a9e27c9254bb28 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 15:21:59 -0500 Subject: [PATCH 32/36] Handle 3.0 call_stmt better --- uncompyle6/parsers/parse30.py | 3 ++- uncompyle6/semantics/customize26_27.py | 1 + uncompyle6/semantics/fragments.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 7baef5a3..7ecd1822 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -17,6 +17,7 @@ class Python30Parser(Python31Parser): assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 come_froms + call_stmt ::= expr _come_froms POP_TOP return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA @@ -158,7 +159,7 @@ class Python30Parser(Python31Parser): ifnotstmt30 ::= testtrue_then _ifstmts_jump30 testfalse_then ::= expr jmp_false_then - testtrue_then ::= expr jmp_true_then + testtrue_then ::= expr jmp_true_then call_stmt ::= expr COME_FROM _ifstmts_jump30 ::= c_stmts POP_TOP diff --git a/uncompyle6/semantics/customize26_27.py b/uncompyle6/semantics/customize26_27.py index 324b50b8..a57920a8 100644 --- a/uncompyle6/semantics/customize26_27.py +++ b/uncompyle6/semantics/customize26_27.py @@ -40,6 +40,7 @@ def customize_for_version26_27(self, version): 'testtrue_then': ( 'not %p', (0, 22) ), }) + # FIXME: this should be a transformation def n_call(node): mapping = self._get_mapping(node) key = node diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 0a56b850..43c4428a 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -790,7 +790,7 @@ class FragmentsWalker(pysource.SourceWalker, object): self.name = code_name # Issue created with later Python code generation is that there - # is a lamda set up with a dummy argument name that is then called + # is a lambda set up with a dummy argument name that is then called # So we can't just translate that as is but need to replace the # dummy name. Below we are picking out the variable name as seen # in the code. And trying to generate code for the other parts From 9874553fb46d21875949b0cc35339a76cbed2edc Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 16:04:16 -0500 Subject: [PATCH 33/36] while1 rule adjustment for 3.0 --- uncompyle6/parsers/parse3.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 880dfd7b..7dcd26bd 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -1542,12 +1542,24 @@ class Python3Parser(PythonParser): for i in range(cfl - 1, first, -1): if tokens[i] != "POP_BLOCK": break - if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"): + if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "BREAK_LOOP"): if not tokens[i].kind.startswith("COME_FROM"): return True # Check that the SETUP_LOOP jumps to the offset after the # COME_FROM_LOOP + + # Python 3.0 has additional: + # JUMP_FORWARD here + # COME_FROM + # POP_TOP + # COME_FROM + # here: + # (target of SETUP_LOOP) + # We won't check this. + if self.version == 3.0: + return False + if 0 <= last < len(tokens) and tokens[last] in ( "COME_FROM_LOOP", "JUMP_BACK", From a3a15414d372867ea677df7d09b0caa5b2471013 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 16:18:39 -0500 Subject: [PATCH 34/36] Add 3.0 whileTrue rule for recent CONTINUE change --- uncompyle6/parsers/parse30.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 7ecd1822..1c66362c 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -23,8 +23,10 @@ class Python30Parser(Python31Parser): compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA # FIXME: combine with parse3.2 - whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK - COME_FROM_LOOP + whileTruestmt ::= SETUP_LOOP l_stmts_opt + JUMP_BACK COME_FROM_LOOP + whileTruestmt ::= SETUP_LOOP l_stmts_opt + CONTINUE COME_FROM_LOOP whileTruestmt ::= SETUP_LOOP returns COME_FROM_LOOP From f1496cad4d09e5df7dee87458d7cd7b415931249 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 17:32:48 -0500 Subject: [PATCH 35/36] 3.0 return_if_stmt and... * Some grammar cleanup * add more grammar debugging --- uncompyle6/parsers/parse26.py | 2 +- uncompyle6/parsers/parse27.py | 2 +- uncompyle6/parsers/parse30.py | 55 ++++++++++++++++++++++++++++------- uncompyle6/parsers/parse34.py | 2 +- uncompyle6/parsers/parse35.py | 2 +- uncompyle6/parsers/parse36.py | 2 +- uncompyle6/parsers/parse37.py | 2 +- uncompyle6/parsers/parse38.py | 2 +- 8 files changed, 52 insertions(+), 17 deletions(-) diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index 9b7c35d4..2d91e9ad 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -467,7 +467,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 2.6: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse27.py b/uncompyle6/parsers/parse27.py index e63f7f69..c4e7b484 100644 --- a/uncompyle6/parsers/parse27.py +++ b/uncompyle6/parsers/parse27.py @@ -292,7 +292,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 2.7: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 1c66362c..e7e88687 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -74,8 +74,8 @@ class Python30Parser(Python31Parser): setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST # Need to keep LOAD_FAST as index 1 - set_comp_func_header ::= BUILD_SET_0 DUP_TOP STORE_FAST - set_comp_func ::= set_comp_func_header + set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST + set_comp_func ::= set_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST @@ -87,7 +87,6 @@ class Python30Parser(Python31Parser): LOAD_FAST FOR_ITER store comp_iter JUMP_BACK _come_froms POP_TOP JUMP_BACK - set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST set_comp ::= set_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK @@ -170,11 +169,9 @@ class Python30Parser(Python31Parser): except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts COME_FROM POP_TOP END_FINALLY - jit_cf_pop ::= JUMP_IF_TRUE _come_froms POP_TOP - jif_cf_pop ::= JUMP_IF_FASLE _come_froms POP_TOP - or ::= expr jit_cf_pop expr come_from_opt - ret_or ::= expr jit_cf_pop expr come_from_opt - ret_and ::= expr jif_cf_pop expr come_from_opt + or ::= expr jmp_true_then expr come_from_opt + ret_or ::= expr jmp_true_then expr come_from_opt + ret_and ::= expr jump_false expr come_from_opt ################################################################################ for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK @@ -185,6 +182,7 @@ class Python30Parser(Python31Parser): POP_TOP END_FINALLY return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP + return_if_stmt ::= ret_expr RETURN_VALUE come_froms POP_TOP and ::= expr jmp_false_then expr come_from_opt @@ -204,8 +202,8 @@ class Python30Parser(Python31Parser): compare_chained2 ::= expr COMPARE_OP RETURN_END_IF """ - def customize_grammar_rules(self, tokens, customize): - super(Python30Parser, self).customize_grammar_rules(tokens, customize) + + def remove_rules_30(self): self.remove_rules(""" iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel @@ -231,16 +229,26 @@ class Python30Parser(Python31Parser): jmp_false ::= POP_JUMP_IF_FALSE jmp_true ::= JUMP_IF_TRUE_OR_POP POP_TOP + jmp_true ::= POP_JUMP_IF_TRUE + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained1 COME_FROM compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained2 COME_FROM ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM + ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF + COME_FROM ret_expr_or_cond + ret_expr_or_cond ::= ret_cond or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM + and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM """) + def customize_grammar_rules(self, tokens, customize): + super(Python30Parser, self).customize_grammar_rules(tokens, customize) + self.remove_rules_30() + self.check_reduce["iflaststmtl"] = "AST" self.check_reduce['ifstmt'] = "AST" self.check_reduce["ifelsestmtc"] = "AST" @@ -312,3 +320,30 @@ class Python30Parser(Python31Parser): class Python30ParserSingle(Python30Parser, PythonParserSingle): pass + +if __name__ == '__main__': + # Check grammar + p = Python30Parser() + p.remove_rules_30() + p.check_grammar() + from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.0: + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() + from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) + opcode_set = set(s.opc.opname).union(set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split())) + remain_tokens = set(tokens) - opcode_set + import re + remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) + remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + remain_tokens = set(remain_tokens) - opcode_set + print(remain_tokens) + import sys + if len(sys.argv) > 1: + from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): + print(rule2str(rule[0])) diff --git a/uncompyle6/parsers/parse34.py b/uncompyle6/parsers/parse34.py index 29bb1598..8eff26dd 100644 --- a/uncompyle6/parsers/parse34.py +++ b/uncompyle6/parsers/parse34.py @@ -72,7 +72,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.4: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 30db4fa7..294c458c 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -263,7 +263,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.5: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index b644806a..5acbc8ec 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -441,7 +441,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.6: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index db004db5..3caaa329 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -177,7 +177,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.7: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index b137385e..47dede7a 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -270,7 +270,7 @@ if __name__ == '__main__': p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY if PYTHON_VERSION == 3.8: - lhs, rhs, tokens, right_recursive = p.check_sets() + lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner s = get_scanner(PYTHON_VERSION, IS_PYPY) opcode_set = set(s.opc.opname).union(set( From df5df9364c243ebfe14a4d2c78f06b0b07bed9c1 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 16 Nov 2019 17:59:30 -0500 Subject: [PATCH 36/36] Grammar debugging for 3.0, 3.7 and 3.8 --- uncompyle6/parsers/parse30.py | 1 + uncompyle6/parsers/parse37.py | 48 +++++++++++++---------------------- uncompyle6/parsers/parse38.py | 32 ++++++++++++++++------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index e7e88687..df932d8a 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -336,6 +336,7 @@ if __name__ == '__main__': LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LAMBDA_MARKER RETURN_LAST """.split())) + ## FIXME: try this remain_tokens = set(tokens) - opcode_set import re remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index 3caaa329..1859ba66 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -141,31 +141,6 @@ class Python37Parser(Python36Parser): """ def customize_grammar_rules(self, tokens, customize): - self.remove_rules(""" - async_forelse_stmt ::= SETUP_LOOP expr - GET_AITER - LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST - YIELD_FROM - store - POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP - LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE - POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK - JUMP_ABSOLUTE END_FINALLY COME_FROM - for_block POP_BLOCK - else_suite COME_FROM_LOOP - stmt ::= async_for_stmt36 - async_for_stmt36 ::= SETUP_LOOP expr - GET_AITER - LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST - YIELD_FROM - store - POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP - LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE - END_FINALLY continues COME_FROM - POP_TOP POP_TOP POP_TOP POP_EXCEPT - POP_TOP POP_BLOCK - COME_FROM_LOOP - """) super(Python37Parser, self).customize_grammar_rules(tokens, customize) class Python37ParserSingle(Python37Parser, PythonParserSingle): @@ -173,22 +148,33 @@ class Python37ParserSingle(Python37Parser, PythonParserSingle): if __name__ == '__main__': # Check grammar + # FIXME: DRY this with other parseXX.py routines p = Python37Parser() p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.7: lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) - opcode_set = set(s.opc.opname).union(set( - """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + opcode_set = set(s.opc.opname).union( + set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME LAMBDA_MARKER RETURN_LAST - """.split())) + """.split() + ) + ) remain_tokens = set(tokens) - opcode_set import re - remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + + remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) - # print(sorted(p.rule2name.items())) + import sys + if len(sys.argv) > 1: + from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): + print(rule2str(rule[0])) diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index 47dede7a..5cea1baf 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -159,7 +159,7 @@ class Python38Parser(Python37Parser): super(Python38Parser, self).__init__(debug_parser) self.customized = {} - def customize_grammar_rules(self, tokens, customize): + def remove_rules_38(self): self.remove_rules(""" stmt ::= async_for_stmt37 stmt ::= for @@ -226,7 +226,10 @@ class Python38Parser(Python37Parser): """) + + def customize_grammar_rules(self, tokens, customize): super(Python37Parser, self).customize_grammar_rules(tokens, customize) + self.remove_rules_38() self.check_reduce['ifstmt'] = 'tokens' self.check_reduce['whileTruestmt38'] = 'tokens' @@ -234,6 +237,7 @@ class Python38Parser(Python37Parser): invalid = super(Python38Parser, self).reduce_is_invalid(rule, ast, tokens, first, last) + self.remove_rules_38() if invalid: return invalid if rule[0] == 'ifstmt': @@ -264,24 +268,34 @@ class Python38Parser(Python37Parser): class Python38ParserSingle(Python38Parser, PythonParserSingle): pass -if __name__ == '__main__': +if __name__ == "__main__": # Check grammar + # FIXME: DRY this with other parseXX.py routines p = Python38Parser() + p.remove_rules_38() p.check_grammar() from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.8: lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets() from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) - opcode_set = set(s.opc.opname).union(set( - """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM - LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME - LAMBDA_MARKER RETURN_LAST + opcode_set = set(s.opc.opname).union( + set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST """.split())) remain_tokens = set(tokens) - opcode_set import re - remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens]) + + remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set print(remain_tokens) - # print(sorted(p.rule2name.items())) + import sys + if len(sys.argv) > 1: + from spark_parser.spark import rule2str + for rule in sorted(p.rule2name.items()): + print(rule2str(rule[0]))