From ebb0342b38a50d7c73519c929e3e7e20e21bca81 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 11 May 2019 08:52:38 -0400 Subject: [PATCH] WIP pypy3.6 handling --- test/Makefile | 2 + test/bytecode_pypy3.6/00_assign.pyc | Bin 0 -> 214 bytes test/bytecode_pypy3.6_run/00_docstring.pyc | Bin 0 -> 2703 bytes test/bytecode_pypy3.6_run/01_fstring.pyc | Bin 0 -> 1880 bytes test/test_pyenvlib.py | 1 + uncompyle6/parsers/parse2.py | 433 +++++++++++++-------- uncompyle6/parsers/parse3.py | 65 +++- uncompyle6/parsers/parse36.py | 16 +- uncompyle6/scanners/scanner3.py | 24 +- uncompyle6/semantics/customize.py | 8 +- uncompyle6/semantics/pysource.py | 147 ++++--- 11 files changed, 450 insertions(+), 246 deletions(-) create mode 100644 test/bytecode_pypy3.6/00_assign.pyc create mode 100644 test/bytecode_pypy3.6_run/00_docstring.pyc create mode 100644 test/bytecode_pypy3.6_run/01_fstring.pyc diff --git a/test/Makefile b/test/Makefile index c4a11f34..c8024f7f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -333,7 +333,9 @@ pypy-3.2 2.4: $(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify #: PyPy 5.0.x with Python 3.6 ... +check-bytecode-pypy3.6: 7.1 7.1: + $(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify diff --git a/test/bytecode_pypy3.6/00_assign.pyc b/test/bytecode_pypy3.6/00_assign.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c711d3b46e583e73fe2b2ddaafaa71bdfc839cf7 GIT binary patch literal 214 zcmZ3$z{};c?RZQ*0|P@c5W@j8kmUfx#VSA|g&~R|g)xdTg(-^Zj8G~=Dq|{B8gmMB zHdC=e33C%;3S%&XCd(HVpi;m5yi~tT9VP|_P39sXp8?8Y`N9a~B!bB#Fq!;?4=CbT zT%1}|l9`|9T2z!@q{#q+r3yfa;>_HFoYeT@{L-T2RQ=+T+>&B_1B3X);^NHoJiUU- ZQVyUZo80`A(wtN~R*(iZAi=^2f&iM;E(QPq literal 0 HcmV?d00001 diff --git a/test/bytecode_pypy3.6_run/00_docstring.pyc b/test/bytecode_pypy3.6_run/00_docstring.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8cd26c8042adf955fc594a6c19910adc925e53a GIT binary patch literal 2703 zcmcguPmj}95Px?5Y}PF4mQoIts7B`C7%d^YEtDd5)vAibq1prVgce!G^Rh7{cKY6P z3$3`&euRDidf)&OCvKH`%9RhWA7r_A<|Uh?G!=3r@Ka{q*l*_je)hbXJQb(Czh(38 zvvna7v2eLG|3+*I{Rd4@+!5%Y`@6W~Ul{G8tR<2puuyWl>k0pM3TZHG{TqJ0{ zB1>R2h!fe*2PhR>f}c~ZWm?I4CH% zyI(ST=pL_8s3^ibiK62s&A-)JDwF0}SE*cuf=6+IH%*75fsAxss6<}YW@z-~PKP}w zc-x~##}!(KKXy2#=5HY!RVBHc#94asx@p9H0#7tHjY~8frtxNr9>o?2n0{;kCCbdS zLg>dDvK(ZuCldpC27!?eO|TzUrh9KI-d8b@a@bX!Ts-H=-r+BI z0=z-<_YK^izws|t#jf~mb0(hvXZw8vUr3cNR|dX8K+i1zXS+1;4mmBU0|CSt{$Dki zzx69CgXdKFYymvmrNMvyZty`6%u5*=p(S?U+cB$g6C38luC7dMlQ4b0K#c9u#CFLE zrq_*2e8KH*7lI;yo84Po4`6R^4~CBhGHVt7_|Nh4@yqe6@n3#OkFL;f&g83=ned_T zXH05}?b1xXF2&^N#nG#y7fV=NTbTvdcP`|U<6Tp1muB(Qp{jiHLE7!Mol-XOSD1E) zZP?v~?PZJpb&tB~%w})-P$pocHW9`qhBSlWcAGk3nwThh*UFvcD;FZPixI_@tzZ4# zj(;77y`2T`v)*Z!n_QO+?h_(+AM>wPlk{MlIv9b*>!+A-R&A!T_6+ zAJA^P@pz_-F1qL+=%POW#*6+JtbNZp#+jBCn$fxEp8K8GcaQX&$eVww!v0@rAsq3a zCxiMke4{_55Fl&;(w3mv+Qm|DR2J^x?stw=gas&dbvp-T+f;W{?p(s+z))sc*g5pf!8?fkhq3S8M8SJQ?P3O& zE~Uzq1-#Eb1@yUpp#3bPG<$sjmY28luxuA##h!%^RsPl7#a&h43Rj1^nnmA_(03Ld z*z-`a?~N=qkJYc?n_{gx5W^IF%u&DRs1NPps08b(2#*E^)QBC-c*JKLP=6!r1=xfJ zt|gfESRNHsNtLm~t*&JJAB?x2i|XSs-k3OIc|6up2h~YtaCQAyg}uOQ#mY)@Xt1I< zbp38*CFEiyI0V9~^p3ZlCd)VQM{lu}|KTgpcPH(54V?j$B! zI(^@XgTRYwY+Oq;G_Ft5R}-4K&7JQmjmmyf{2%(-y4I+2TP>DtmJUl``GVtZ!4-Ag zYCF8mo8uI1Bgb{GC|};Z%Hpz>HxZIIUDU-M*HdGsOBQ87Vz))c{RfvdSsE#D3?l1sO+_DCmcd+GzK?TTeIY zo6XMT=4-uUzO-0Hu^T@oJTWA+@DWOfve82;wZxeppc3Qn05s6SxR8*7^c!(1z7gm0 zQsM`YDT}T=*Ri(j-k98)%~18iX0s}JU?SypnBY*w{m{3Lj;x1PXJtAJZ?jS*G*Qp- zjh0Y!G0T4OOrFvYL9moV;}`7*19NFSK07BAy~YeKhi6EG#zRo6GcTRn^U|kz-S*rl zc7ns+?iW-9S0odvWztf|uWfsQ)AFKSRHj(YSY4@3Smd5x1+I_nQ#&13c@W*ckO_s% zV8{QQ${sGiqRT@0W z_<4*A82QU6Qeo@{{G$Z_>IWPPE0e8yUB$TIoHYwi;K#hsW zAmYyG8>3Hvk0)cwRrFf%%Q5{CTJ4DR@ChZLv1c^hju&*BVbJ#>;&EjvLZX7mz3u$> ziOuc90Q#QVrSQ=+6b903$VF+&qAaCMSwOuY3xQl? ICYv?ozmr|a2><{9 literal 0 HcmV?d00001 diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index e406026c..e1ffecc6 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -43,6 +43,7 @@ TEST_VERSIONS = ( "pypy3.5-5.7.1-beta", "pypy3.5-5.9.0", "pypy3.5-6.0.0", + "pypy3.6-7.1.0", "native", ) + tuple(python_versions) diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 5da89ba5..d9bed6c4 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2018 Rocky Bernstein +# Copyright (c) 2015-2019 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # Copyright (c) 1999 John Aycock @@ -31,10 +31,10 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parsers.treenode import SyntaxTree from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -class Python2Parser(PythonParser): +class Python2Parser(PythonParser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): - super(Python2Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser) + super(Python2Parser, self).__init__(SyntaxTree, "stmts", debug=debug_parser) self.new_rules = set() def p_print2(self, args): @@ -52,7 +52,7 @@ class Python2Parser(PythonParser): """ def p_print_to(self, args): - ''' + """ stmt ::= print_to stmt ::= print_to_nl stmt ::= print_nl_to @@ -62,10 +62,10 @@ class Python2Parser(PythonParser): print_to_items ::= print_to_items print_to_item print_to_items ::= print_to_item print_to_item ::= DUP_TOP expr ROT_TWO PRINT_ITEM_TO - ''' + """ def p_grammar(self, args): - ''' + """ sstmt ::= stmt sstmt ::= return RETURN_LAST @@ -176,12 +176,12 @@ class Python2Parser(PythonParser): jmp_abs ::= JUMP_ABSOLUTE jmp_abs ::= JUMP_BACK jmp_abs ::= CONTINUE - ''' + """ def p_generator_exp2(self, args): - ''' + """ generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 - ''' + """ def p_expr2(self, args): """ @@ -252,25 +252,41 @@ class Python2Parser(PythonParser): this. """ - if 'PyPy' in customize: + if "PyPy" in customize: # PyPy-specific customizations - self.addRule(""" + self.addRule( + """ stmt ::= assign3_pypy stmt ::= assign2_pypy assign3_pypy ::= expr expr expr store store store assign2_pypy ::= expr expr store store list_comp ::= expr BUILD_LIST_FROM_ARG for_iter store list_iter JUMP_BACK - """, nop_func) + """, + nop_func, + ) # For a rough break out on the first word. This may # include instructions that don't need customization, # but we'll do a finer check after the rough breakout. customize_instruction_basenames = frozenset( - ('BUILD', 'CALL', 'CONTINUE', 'DELETE', - 'DUP', 'EXEC', 'GET', 'JUMP', - 'LOAD', 'LOOKUP', 'MAKE', 'SETUP', - 'RAISE', 'UNPACK')) + ( + "BUILD", + "CALL", + "CONTINUE", + "DELETE", + "DUP", + "EXEC", + "GET", + "JUMP", + "LOAD", + "LOOKUP", + "MAKE", + "SETUP", + "RAISE", + "UNPACK", + ) + ) # Opcode names in the custom_seen_ops set have rules that get added # unconditionally and the rules are constant. So they need to be done @@ -284,139 +300,191 @@ class Python2Parser(PythonParser): # Do a quick breakout before testing potentially # each of the dozen or so instruction in if elif. - if (opname[:opname.find('_')] not in customize_instruction_basenames - or opname in custom_seen_ops): + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_seen_ops + ): continue - opname_base = opname[:opname.rfind('_')] + opname_base = opname[: opname.rfind("_")] # The order of opname listed is roughly sorted below - if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'): + if opname_base in ("BUILD_LIST", "BUILD_SET", "BUILD_TUPLE"): # We do this complicated test to speed up parsing of # pathelogically long literals, especially those over 1024. build_count = token.attr - thousands = (build_count//1024) - thirty32s = ((build_count//32) % 32) + thousands = build_count // 1024 + thirty32s = (build_count // 32) % 32 if thirty32s > 0: - rule = "expr32 ::=%s" % (' expr' * 32) + rule = "expr32 ::=%s" % (" expr" * 32) self.add_unique_rule(rule, opname_base, build_count, customize) if thousands > 0: - self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), - opname_base, build_count, customize) - collection = opname_base[opname_base.find('_')+1:].lower() - rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + - 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) - self.add_unique_rules([ - "expr ::= %s" % collection, - rule], customize) + self.add_unique_rule( + "expr1024 ::=%s" % (" expr32" * 32), + opname_base, + build_count, + customize, + ) + collection = opname_base[opname_base.find("_") + 1 :].lower() + rule = ( + ("%s ::= " % collection) + + "expr1024 " * thousands + + "expr32 " * thirty32s + + "expr " * (build_count % 32) + + opname + ) + self.add_unique_rules(["expr ::= %s" % collection, rule], customize) continue - elif opname_base == 'BUILD_MAP': - if opname == 'BUILD_MAP_n': + elif opname_base == "BUILD_MAP": + if opname == "BUILD_MAP_n": # PyPy sometimes has no count. Sigh. - self.add_unique_rules([ - 'kvlist_n ::= kvlist_n kv3', - 'kvlist_n ::=', - 'dict ::= BUILD_MAP_n kvlist_n', - ], customize) + self.add_unique_rules( + [ + "kvlist_n ::= kvlist_n kv3", + "kvlist_n ::=", + "dict ::= BUILD_MAP_n kvlist_n", + ], + customize, + ) if self.version >= 2.7: self.add_unique_rule( - 'dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store ' - 'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST', - 'dict_comp_func', 0, customize) + "dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store " + "comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST", + "dict_comp_func", + 0, + customize, + ) else: - kvlist_n = ' kv3' * token.attr + kvlist_n = " kv3" * token.attr rule = "dict ::= %s%s" % (opname, kvlist_n) self.addRule(rule, nop_func) continue - elif opname_base == 'BUILD_SLICE': - slice_num = token.attr + elif opname_base == "BUILD_SLICE": + slice_num = token.attr if slice_num == 2: - self.add_unique_rules([ - 'expr ::= build_slice2', - 'build_slice2 ::= expr expr BUILD_SLICE_2' - ], customize) + self.add_unique_rules( + [ + "expr ::= build_slice2", + "build_slice2 ::= expr expr BUILD_SLICE_2", + ], + customize, + ) else: - assert slice_num == 3, ("BUILD_SLICE value must be 2 or 3; is %s" % - slice_num) - self.add_unique_rules([ - 'expr ::= build_slice3', - 'build_slice3 ::= expr expr expr BUILD_SLICE_3', - ], customize) + assert slice_num == 3, ( + "BUILD_SLICE value must be 2 or 3; is %s" % slice_num + ) + self.add_unique_rules( + [ + "expr ::= build_slice3", + "build_slice3 ::= expr expr expr BUILD_SLICE_3", + ], + customize, + ) continue - elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', - 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): + elif opname_base in ( + "CALL_FUNCTION", + "CALL_FUNCTION_VAR", + "CALL_FUNCTION_VAR_KW", + "CALL_FUNCTION_KW", + ): args_pos, args_kw = self.get_pos_kw(token) # number of apply equiv arguments: - nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3 - rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ - + 'expr ' * nak + opname - elif opname_base == 'CALL_METHOD': + nak = (len(opname_base) - len("CALL_FUNCTION")) // 3 + rule = ( + "call ::= expr " + + "expr " * args_pos + + "kwarg " * args_kw + + "expr " * nak + + opname + ) + elif opname_base == "CALL_METHOD": # PyPy only - DRY with parse3 args_pos, args_kw = self.get_pos_kw(token) # number of apply equiv arguments: - nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 - rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ - + 'expr ' * nak + opname - elif opname == 'CONTINUE_LOOP': - self.addRule('continue ::= CONTINUE_LOOP', nop_func) + nak = (len(opname_base) - len("CALL_METHOD")) // 3 + rule = ( + "call ::= expr " + + "expr " * args_pos + + "kwarg " * args_kw + + "expr " * nak + + opname + ) + elif opname == "CONTINUE_LOOP": + self.addRule("continue ::= CONTINUE_LOOP", nop_func) custom_seen_ops.add(opname) continue - elif opname == 'DELETE_ATTR': - self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func) + elif opname == "DELETE_ATTR": + self.addRule("del_stmt ::= expr DELETE_ATTR", nop_func) custom_seen_ops.add(opname) continue - elif opname.startswith('DELETE_SLICE'): - self.addRule(""" + elif opname.startswith("DELETE_SLICE"): + self.addRule( + """ del_expr ::= expr del_stmt ::= del_expr DELETE_SLICE+0 del_stmt ::= del_expr del_expr DELETE_SLICE+1 del_stmt ::= del_expr del_expr DELETE_SLICE+2 del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3 - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) - self.check_reduce['del_expr'] = 'AST' + self.check_reduce["del_expr"] = "AST" continue - elif opname == 'DELETE_DEREF': - self.addRule(""" + elif opname == "DELETE_DEREF": + self.addRule( + """ stmt ::= del_deref_stmt del_deref_stmt ::= DELETE_DEREF - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname == 'DELETE_SUBSCR': - self.addRule(""" + elif opname == "DELETE_SUBSCR": + self.addRule( + """ del_stmt ::= delete_subscript delete_subscript ::= expr expr DELETE_SUBSCR - """, nop_func) - self.check_reduce['delete_subscript'] = 'AST' + """, + nop_func, + ) + self.check_reduce["delete_subscript"] = "AST" custom_seen_ops.add(opname) continue - elif opname == 'GET_ITER': - self.addRule(""" + elif opname == "GET_ITER": + self.addRule( + """ expr ::= get_iter attribute ::= expr GET_ITER - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'): + elif opname_base in ("DUP_TOPX", "RAISE_VARARGS"): # FIXME: remove these conditions if they are not needed. # no longer need to add a rule continue - elif opname == 'EXEC_STMT': - self.addRule(""" + elif opname == "EXEC_STMT": + self.addRule( + """ stmt ::= exec_stmt exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT exprlist ::= expr+ - """, nop_func) + """, + nop_func, + ) continue - elif opname == 'JUMP_IF_NOT_DEBUG': - self.addRule(""" + elif opname == "JUMP_IF_NOT_DEBUG": + self.addRule( + """ jmp_true_false ::= POP_JUMP_IF_TRUE jmp_true_false ::= POP_JUMP_IF_FALSE stmt ::= assert_pypy @@ -426,107 +494,152 @@ class Python2Parser(PythonParser): assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM - """, nop_func) + """, + nop_func, + ) continue - elif opname == 'LOAD_ATTR': - self.addRule(""" + elif opname == "LOAD_ATTR": + self.addRule( + """ expr ::= attribute attribute ::= expr LOAD_ATTR - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname == 'LOAD_LISTCOMP': + elif opname == "LOAD_LISTCOMP": self.addRule("expr ::= listcomp", nop_func) custom_seen_ops.add(opname) continue - elif opname == 'LOAD_SETCOMP': - self.add_unique_rules([ - "expr ::= set_comp", - "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1" - ], customize) + elif opname == "LOAD_SETCOMP": + self.add_unique_rules( + [ + "expr ::= set_comp", + "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1", + ], + customize, + ) custom_seen_ops.add(opname) continue - elif opname == 'LOOKUP_METHOD': + elif opname == "LOOKUP_METHOD": # A PyPy speciality - DRY with parse3 - self.addRule(""" + self.addRule( + """ expr ::= attribute attribute ::= expr LOOKUP_METHOD """, - nop_func) + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname_base == 'MAKE_FUNCTION': - if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': - self.addRule('mklambda ::= %s LOAD_LAMBDA %s' % - ('pos_arg ' * token.attr, opname), nop_func) - rule = 'mkfunc ::= %s LOAD_CODE %s' % ('expr ' * token.attr, opname) - elif opname_base == 'MAKE_CLOSURE': + elif opname_base == "MAKE_FUNCTION": + if i > 0 and tokens[i - 1] == "LOAD_LAMBDA": + self.addRule( + "mklambda ::= %s LOAD_LAMBDA %s" + % ("pos_arg " * token.attr, opname), + nop_func, + ) + rule = "mkfunc ::= %s LOAD_CODE %s" % ("expr " * token.attr, opname) + elif opname_base == "MAKE_CLOSURE": # FIXME: use add_unique_rules to tidy this up. - if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': - self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' % - ('expr ' * token.attr, opname), nop_func) + if i > 0 and tokens[i - 1] == "LOAD_LAMBDA": + self.addRule( + "mklambda ::= %s load_closure LOAD_LAMBDA %s" + % ("expr " * token.attr, opname), + nop_func, + ) if i > 0: - prev_tok = tokens[i-1] - if prev_tok == 'LOAD_GENEXPR': - self.add_unique_rules([ - ('generator_exp ::= %s load_closure LOAD_GENEXPR %s expr' - ' GET_ITER CALL_FUNCTION_1' % - ('expr ' * token.attr, opname))], customize) + prev_tok = tokens[i - 1] + if prev_tok == "LOAD_GENEXPR": + self.add_unique_rules( + [ + ( + "generator_exp ::= %s load_closure LOAD_GENEXPR %s expr" + " GET_ITER CALL_FUNCTION_1" + % ("expr " * token.attr, opname) + ) + ], + customize, + ) pass - self.add_unique_rules([ - ('mkfunc ::= %s load_closure LOAD_CODE %s' % - ('expr ' * token.attr, opname))], customize) + self.add_unique_rules( + [ + ( + "mkfunc ::= %s load_closure LOAD_CODE %s" + % ("expr " * token.attr, opname) + ) + ], + customize, + ) if self.version >= 2.7: if i > 0: - prev_tok = tokens[i-1] - if prev_tok == 'LOAD_DICTCOMP': - self.add_unique_rules([ - ('dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr' - ' GET_ITER CALL_FUNCTION_1' % - ('expr ' * token.attr, opname))], customize) - elif prev_tok == 'LOAD_SETCOMP': - self.add_unique_rules([ - "expr ::= set_comp", - ('set_comp ::= %s load_closure LOAD_SETCOMP %s expr' - ' GET_ITER CALL_FUNCTION_1' % - ('expr ' * token.attr, opname)) - ], customize) + prev_tok = tokens[i - 1] + if prev_tok == "LOAD_DICTCOMP": + self.add_unique_rules( + [ + ( + "dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr" + " GET_ITER CALL_FUNCTION_1" + % ("expr " * token.attr, opname) + ) + ], + customize, + ) + elif prev_tok == "LOAD_SETCOMP": + self.add_unique_rules( + [ + "expr ::= set_comp", + ( + "set_comp ::= %s load_closure LOAD_SETCOMP %s expr" + " GET_ITER CALL_FUNCTION_1" + % ("expr " * token.attr, opname) + ), + ], + customize, + ) pass pass continue - elif opname == 'SETUP_EXCEPT': - if 'PyPy' in customize: - self.add_unique_rules([ - "stmt ::= try_except_pypy", - "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy", - "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM" - ], customize) + elif opname == "SETUP_EXCEPT": + if "PyPy" in customize: + self.add_unique_rules( + [ + "stmt ::= try_except_pypy", + "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy", + "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM", + ], + customize, + ) custom_seen_ops.add(opname) continue - elif opname == 'SETUP_FINALLY': - if 'PyPy' in customize: - self.addRule(""" + elif opname == "SETUP_FINALLY": + if "PyPy" in customize: + self.addRule( + """ stmt ::= tryfinallystmt_pypy tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY - suite_stmts_opt END_FINALLY""", nop_func) + suite_stmts_opt END_FINALLY""", + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): + elif opname_base in ("UNPACK_TUPLE", "UNPACK_SEQUENCE"): custom_seen_ops.add(opname) - rule = 'unpack ::= ' + opname + ' store' * token.attr - elif opname_base == 'UNPACK_LIST': + rule = "unpack ::= " + opname + " store" * token.attr + elif opname_base == "UNPACK_LIST": custom_seen_ops.add(opname) - rule = 'unpack_list ::= ' + opname + ' store' * token.attr + rule = "unpack_list ::= " + opname + " store" * token.attr else: continue self.addRule(rule, nop_func) pass - self.check_reduce['raise_stmt1'] = 'tokens' - self.check_reduce['aug_assign2'] = 'AST' - self.check_reduce['or'] = 'AST' + self.check_reduce["raise_stmt1"] = "tokens" + self.check_reduce["aug_assign2"] = "AST" + self.check_reduce["or"] = "AST" # self.check_reduce['_stmts'] = 'AST' # Dead code testing... @@ -541,24 +654,30 @@ class Python2Parser(PythonParser): # Dead code testing... # if lhs == 'while1elsestmt': # from trepan.api import debug; debug() - if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'): + if ( + lhs in ("aug_assign1", "aug_assign2") + and ast[0] + and ast[0][0] in ("and", "or") + ): return True - elif lhs in ('raise_stmt1',): + elif lhs in ("raise_stmt1",): # We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule - return (tokens[first] == 'LOAD_ASSERT' and (last >= len(tokens))) - elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')): + return tokens[first] == "LOAD_ASSERT" and (last >= len(tokens)) + elif rule == ("or", ("expr", "jmp_true", "expr", "\\e_come_from_opt")): expr2 = ast[2] - return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT' - elif lhs in ('delete_subscript', 'del_expr'): + return expr2 == "expr" and expr2[0] == "LOAD_ASSERT" + elif lhs in ("delete_subscript", "del_expr"): op = ast[0][0] - return op.kind in ('and', 'or') + return op.kind in ("and", "or") return False + class Python2ParserSingle(Python2Parser, PythonParserSingle): pass -if __name__ == '__main__': + +if __name__ == "__main__": # Check grammar p = Python2Parser() p.check_grammar() diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index eaa5d439..a39dc570 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -432,7 +432,7 @@ class Python3Parser(PythonParser): else: return "%s_0" % (token.kind) - def custom_build_class_rule(self, opname, i, token, tokens, customize): + def custom_build_class_rule(self, opname, i, token, tokens, customize, is_pypy): """ # Should the first rule be somehow folded into the 2nd one? build_class ::= LOAD_BUILD_CLASS mkfunc @@ -483,10 +483,18 @@ class Python3Parser(PythonParser): call_function = call_fn_tok.kind if call_function.startswith("CALL_FUNCTION_KW"): self.addRule("classdef ::= build_class_kw store", nop_func) - rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" % ( - "expr " * (call_fn_tok.attr - 1), - call_function, - ) + if is_pypy: + args_pos, args_kw = self.get_pos_kw(call_fn_tok) + rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %s%s%s" % ( + "expr " * (args_pos - 1), + "kwarg " * (args_kw), + call_function, + ) + else: + rule = ( + "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" + % ("expr " * (call_fn_tok.attr - 1), call_function) + ) else: call_function = self.call_fn_name(call_fn_tok) rule = "build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" % ( @@ -496,7 +504,7 @@ class Python3Parser(PythonParser): self.addRule(rule, nop_func) return - def custom_classfunc_rule(self, opname, token, customize, next_token): + def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy): """ call ::= expr {expr}^n CALL_FUNCTION_n call ::= expr {expr}^n CALL_FUNCTION_VAR_n @@ -514,18 +522,28 @@ class Python3Parser(PythonParser): # Yes, this computation based on instruction name is a little bit hoaky. nak = (len(opname) - len("CALL_FUNCTION")) // 3 - token.kind = self.call_fn_name(token) uniq_param = args_kw + args_pos # Note: 3.5+ have subclassed this method; so we don't handle # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here. - rule = ( - "call ::= expr " - + ("pos_arg " * args_pos) - + ("kwarg " * args_kw) - + "expr " * nak - + token.kind - ) + if is_pypy and self.version >= 3.6: + if token == "CALL_FUNCTION": + token.kind = self.call_fn_name(token) + rule = ( + "call ::= expr " + + ("pos_arg " * args_pos) + + ("kwarg " * args_kw) + + token.kind + ) + else: + token.kind = self.call_fn_name(token) + rule = ( + "call ::= expr " + + ("pos_arg " * args_pos) + + ("kwarg " * args_kw) + + "expr " * nak + + token.kind + ) self.add_unique_rule(rule, token.kind, uniq_param, customize) @@ -821,7 +839,9 @@ class Python3Parser(PythonParser): """ self.addRule(rule, nop_func) - self.custom_classfunc_rule(opname, token, customize, tokens[i + 1]) + self.custom_classfunc_rule( + opname, token, customize, tokens[i + 1], is_pypy + ) # Note: don't add to custom_ops_processed. elif opname_base == "CALL_METHOD": @@ -880,21 +900,30 @@ class Python3Parser(PythonParser): self.addRule( """ stmt ::= assert_pypy - stmt ::= assert2_pypy", nop_func) + stmt ::= assert_not_pypy + stmt ::= assert2_pypy + stmt ::= assert2_not_pypy assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM + assert_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false + LOAD_ASSERT RAISE_VARARGS_1 COME_FROM assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 - RAISE_VARARGS_1 COME_FROM, + RAISE_VARARGS_1 COME_FROM + assert2_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM """, nop_func, ) custom_ops_processed.add(opname) elif opname == "LOAD_BUILD_CLASS": - self.custom_build_class_rule(opname, i, token, tokens, customize) + self.custom_build_class_rule( + opname, i, token, tokens, customize, is_pypy + ) # Note: don't add to custom_ops_processed. elif opname == "LOAD_CLASSDEREF": # Python 3.4+ diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index bdc65bca..d0518bbd 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -316,7 +316,7 @@ class Python36Parser(Python35Parser): pass return - def custom_classfunc_rule(self, opname, token, customize, next_token): + def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy): args_pos, args_kw = self.get_pos_kw(token) @@ -338,10 +338,14 @@ class Python36Parser(Python35Parser): self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize) if opname.startswith('CALL_FUNCTION_KW'): - self.addRule("expr ::= call_kw36", nop_func) - values = 'expr ' * token.attr - rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) - self.add_unique_rule(rule, token.kind, token.attr, customize) + if is_pypy: + # PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions + super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy) + else: + self.addRule("expr ::= call_kw36", nop_func) + values = 'expr ' * token.attr + rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) + self.add_unique_rule(rule, token.kind, token.attr, customize) elif opname == 'CALL_FUNCTION_EX_KW': # Note: this doesn't exist in 3.7 and later self.addRule("""expr ::= call_ex_kw4 @@ -406,7 +410,7 @@ class Python36Parser(Python35Parser): """, nop_func) pass else: - super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token) + super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy) def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python36Parser, diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index f87ad8a0..6a025486 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -260,7 +260,10 @@ class Scanner3(Scanner): # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n else: - assert_can_follow = inst.opname == "POP_JUMP_IF_TRUE" and i + 1 < n + assert_can_follow = ( + inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE") + and i + 1 < n + ) if assert_can_follow: next_inst = self.insts[i + 1] if ( @@ -270,9 +273,7 @@ class Scanner3(Scanner): ): raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] raise_inst = self.insts[raise_idx] - if raise_inst.opname.startswith( - "RAISE_VARARGS" - ): + if raise_inst.opname.startswith("RAISE_VARARGS"): self.load_asserts.add(next_inst.offset) pass pass @@ -428,11 +429,16 @@ class Scanner3(Scanner): else: opname = "%s_%d" % (opname, pos_args) - elif self.is_pypy and opname == "JUMP_IF_NOT_DEBUG": - # The value in the dict is in special cases in semantic actions, such - # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put - # in arbitrary value 0. - customize[opname] = 0 + elif self.is_pypy and opname in ("JUMP_IF_NOT_DEBUG", "CALL_FUNCTION"): + if opname == "JUMP_IF_NOT_DEBUG": + # The value in the dict is in special cases in semantic actions, such + # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put + # in arbitrary value 0. + customize[opname] = 0 + elif self.version >= 3.6 and argval > 255: + opname = "CALL_FUNCTION_KW" + pass + elif opname == "UNPACK_EX": # FIXME: try with scanner and parser by # changing argval diff --git a/uncompyle6/semantics/customize.py b/uncompyle6/semantics/customize.py index 95ad4b19..9f2a0149 100644 --- a/uncompyle6/semantics/customize.py +++ b/uncompyle6/semantics/customize.py @@ -28,8 +28,12 @@ def customize_for_version(self, is_pypy, version): # PyPy changes ####################### TABLE_DIRECT.update({ - 'assert_pypy': ( '%|assert %c\n' , 1 ), - 'assert2_pypy': ( '%|assert %c, %c\n' , 1, 4 ), + 'assert_pypy': ( '%|assert %c\n' , (1, 'assert_expr') ), + 'assert_not_pypy': ( '%|assert not %c\n' , (1, 'assert_exp') ), + 'assert2_not_pypy': ( '%|assert not %c, %c\n' , (1, 'assert_exp'), + (4, 'expr') ), + 'assert2_pypy': ( '%|assert %c, %c\n' , (1, 'assert_expr'), + (4, 'expr') ), 'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ), 'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ), 'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ), diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 7690fd64..b78c26cf 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1566,6 +1566,15 @@ class SourceWalker(GenericASTTraversal, object): if node[n].kind.startswith("CALL_FUNCTION_KW"): # 3.6+ starts doing this kwargs = node[n - 1].attr + if self.is_pypy: + # FIXME: this doesn't handle positional and keyword args + # properly. Need to do something more like that below + # in the non-PYPY 3.6 case. + self.template_engine(('(%[0]{attr}=%c)', 1), node[n-1]) + return + else: + kwargs = node[n-1].attr + assert isinstance(kwargs, tuple) i = n - (len(kwargs) + 1) j = 1 + n - node[n].attr @@ -1750,65 +1759,95 @@ class SourceWalker(GenericASTTraversal, object): else: kv_node = node[1:] else: - assert node[-1].kind.startswith("kvlist") - kv_node = node[-1] - - first_time = True - for kv in kv_node: - assert kv in ("kv", "kv2", "kv3") - - # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR - # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR - # kv3 ::= expr expr STORE_MAP - - # FIXME: DRY this and the above indent = self.indent + " " - if kv == "kv": - self.write(sep) - name = self.traverse(kv[-2], indent="") - if first_time: - line_number = self.indent_if_source_nl(line_number, indent) - first_time = False + line_number = self.line_number + sep = '' + opname = node[-1].kind + if self.is_pypy and self.version >= 3.6: + if opname.startswith('BUILD_CONST_KEY_MAP'): + keys = node[-2].attr + # FIXME: DRY this and the above + for i in range(len(keys)): + key = keys[i] + value = self.traverse(node[i], indent='') + self.write(sep, key, ': ', value) + sep = ", " + if line_number != self.line_number: + sep += "\n" + self.indent + " " + line_number = self.line_number + pass + pass pass - line_number = self.line_number - self.write(name, ": ") - value = self.traverse( - kv[1], indent=self.indent + (len(name) + 2) * " " - ) - elif kv == "kv2": - self.write(sep) - name = self.traverse(kv[1], indent="") - if first_time: - line_number = self.indent_if_source_nl(line_number, indent) - first_time = False + else: + if opname.startswith('kvlist'): + list_node = node[0] + else: + list_node = node + + assert list_node[-1].kind.startswith('BUILD_MAP') + for i in range(0, len(list_node)-1, 2): + key = self.traverse(list_node[i], indent='') + value = self.traverse(list_node[i+1], indent='') + self.write(sep, key, ': ', value) + sep = ", " + if line_number != self.line_number: + sep += "\n" + self.indent + " " + line_number = self.line_number + pass + pass pass - line_number = self.line_number - self.write(name, ": ") - value = self.traverse( - kv[-3], indent=self.indent + (len(name) + 2) * " " - ) - elif kv == "kv3": - self.write(sep) - name = self.traverse(kv[-2], indent="") - if first_time: - line_number = self.indent_if_source_nl(line_number, indent) - first_time = False + elif opname.startswith('kvlist'): + kv_node = node[-1] + first_time = True + for kv in kv_node: + assert kv in ('kv', 'kv2', 'kv3') + + # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR + # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR + # kv3 ::= expr expr STORE_MAP + + # FIXME: DRY this and the above + if kv == 'kv': + self.write(sep) + name = self.traverse(kv[-2], indent='') + if first_time: + line_number = self.indent_if_source_nl(line_number, indent) + first_time = False + pass + line_number = self.line_number + self.write(name, ': ') + value = self.traverse(kv[1], indent=self.indent+(len(name)+2)*' ') + elif kv == 'kv2': + self.write(sep) + name = self.traverse(kv[1], indent='') + if first_time: + line_number = self.indent_if_source_nl(line_number, indent) + first_time = False + pass + line_number = self.line_number + self.write(name, ': ') + value = self.traverse(kv[-3], indent=self.indent+(len(name)+2)*' ') + elif kv == 'kv3': + self.write(sep) + name = self.traverse(kv[-2], indent='') + if first_time: + line_number = self.indent_if_source_nl(line_number, indent) + first_time = False + pass + line_number = self.line_number + self.write(name, ': ') + line_number = self.line_number + value = self.traverse(kv[0], indent=self.indent+(len(name)+2)*' ') + pass + self.write(value) + sep = ", " + if line_number != self.line_number: + sep += "\n" + self.indent + " " + line_number = self.line_number + pass pass - line_number = self.line_number - self.write(name, ": ") - line_number = self.line_number - value = self.traverse( - kv[0], indent=self.indent + (len(name) + 2) * " " - ) - pass - self.write(value) - sep = ", " - if line_number != self.line_number: - sep += "\n" + self.indent + " " - line_number = self.line_number - pass + pass - pass if sep.startswith(",\n"): self.write(sep[1:]) if node[0] != "dict_entry":