From a3e10db8dc231b3afebaff02744bc86437dabe46 Mon Sep 17 00:00:00 2001 From: rocky Date: Mon, 25 Jul 2016 13:05:54 -0400 Subject: [PATCH] Handle PyPy CALL_METHOD op more correctly Start testing pypy2.7 and 3.2 bytecodes --- __pkginfo__.py | 2 +- requirements.txt | 2 +- test/Makefile | 12 ++++++-- test/bytecode_pypy2.7/01_fns.pyc | Bin 0 -> 155 bytes test/bytecode_pypy2.7/02_closure.pyc | Bin 0 -> 348 bytes test/bytecode_pypy2.7/02_complex.pyc | Bin 0 -> 154 bytes test/bytecode_pypy2.7/02_def.pyc | Bin 0 -> 624 bytes test/bytecode_pypy2.7/02_slice.pyc | Bin 0 -> 176 bytes test/bytecode_pypy2.7/05_setattr.pyc | Bin 0 -> 357 bytes test/bytecode_pypy3.2/05_setattr.pyc | Bin 0 -> 402 bytes .../06_setif_comprehension.pyc | Bin 0 -> 438 bytes test/simple_source/bug27+/05_setattr.py | 9 ++++++ uncompyle6/parsers/parse2.py | 26 +++++++++++------- uncompyle6/parsers/parse3.py | 17 +++++++++--- uncompyle6/scanners/scanner2.py | 7 +++-- uncompyle6/scanners/scanner27.py | 6 +++- uncompyle6/scanners/scanner3.py | 18 ++++++------ 17 files changed, 70 insertions(+), 29 deletions(-) create mode 100644 test/bytecode_pypy2.7/01_fns.pyc create mode 100644 test/bytecode_pypy2.7/02_closure.pyc create mode 100644 test/bytecode_pypy2.7/02_complex.pyc create mode 100644 test/bytecode_pypy2.7/02_def.pyc create mode 100644 test/bytecode_pypy2.7/02_slice.pyc create mode 100644 test/bytecode_pypy2.7/05_setattr.pyc create mode 100644 test/bytecode_pypy3.2/05_setattr.pyc create mode 100644 test/bytecode_pypy3.2/06_setif_comprehension.pyc create mode 100644 test/simple_source/bug27+/05_setattr.py diff --git a/__pkginfo__.py b/__pkginfo__.py index c6126bc9..0c03e501 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -37,7 +37,7 @@ entry_points={ ]} ftp_url = None install_requires = ['spark-parser >= 1.4.0', - 'xdis >= 2.0.1'] + 'xdis >= 2.0.2'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' modname = 'uncompyle6' diff --git a/requirements.txt b/requirements.txt index 5add4966..936d59fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ spark-parser >= 1.2.1 -xdis >= 2.0.1 +xdis >= 2.0.2 diff --git a/test/Makefile b/test/Makefile index ed53cac0..4247f2a0 100644 --- a/test/Makefile +++ b/test/Makefile @@ -110,9 +110,17 @@ check-3.2-ok: check-3.4-ok: $(PYTHON) test_pythonlib.py --ok-3.4 --verify $(COMPILE) -#:PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4] +#: PyPy of some sort. E.g. [PyPy 5.0.1 with GCC 4.8.4] # Skip for now -2.4 2.6 5.0: +2.6: + +#: PyPy 5.0.x with ... +5.0: + $(PYTHON) test_pythonlib.py --bytecode-pypy2.7 --verify + +#: PyPy 2.4.x with ... +2.4: + $(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify clean: clean-py-dis clean-dis clean-unverified diff --git a/test/bytecode_pypy2.7/01_fns.pyc b/test/bytecode_pypy2.7/01_fns.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2b2d8ef41604e0ce2c75506d7f5be963fa3142c7 GIT binary patch literal 155 zcmd=3%**vRtRXC!0SXwQbg>YSk;=f(%)r2y$_S=X7#M;zK=OVXOhB@P9Z00-NO z6qm38**W>iK&A$e#eg76WP#k`%-n*U)cE53(xT*4eW0qGVtoU{__VxYy@JXT4xkE~ S-29Z%oK!oI@x>t1*%<*Acpb3- literal 0 HcmV?d00001 diff --git a/test/bytecode_pypy2.7/02_closure.pyc b/test/bytecode_pypy2.7/02_closure.pyc new file mode 100644 index 0000000000000000000000000000000000000000..174202c0ff61cb054558312d46f74ee886373381 GIT binary patch literal 348 zcmYk1O^O0B6oe}}pa?Rf9>A?Dm01sS1Q#v1a3h#bi_#kDqz4AvDgGS8qj?4|;H$J~ z2U4k*_f>ux_h#dEJx?^ALp@j56cYmnU@Aso+^fjWi(^rh={h1OAW?*lz$L^AdZc+A zCvlPi-vPFbGazOK?0Gn$QBeOOHZ_TVR~s9fqN+(x+s&WJd`~QVSHvyX+tkdQ%`o-)rDU$Qtp1?sa>yewi_3bI*b>f(OZ MHqKM?8-i>-0Lc6t8I8P0~9bq>0&M*BZYw>3P>_C1Zyw>xs^~kC?(?HrvXx40wO9jK%xku zL=DI-&de>yNsTYgFD*(=)laP`C`v6Z&dkr#H!zA%&Iid==oM6!Z~&FsM{H>i!)M3rpqGDrY}o;fPK|OBPCmBG)3P>7;1Mg5N;27Tp#dkn7Mq zdf@geRPo}IhYt$qJVIs`aiE)!zi`fP(rps2e75_w@zBLT!_j!f(_nkZVQ@b}zZb8{ zLRaJNE-1(mzUvJQFR1{&!dVI2v|0b3YG9)gI)1F7RPPG?L$+K`8Ea7D+8U8|eqw;NKdYt?n09+$TPKq?y|#@f3|7 LiBN|@U-$I|Nq{?H literal 0 HcmV?d00001 diff --git a/test/bytecode_pypy2.7/02_slice.pyc b/test/bytecode_pypy2.7/02_slice.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37142bddec1f031386c1c9fb9cdca8ead3f1da15 GIT binary patch literal 176 zcmd=3%*&PcqarMs0ScI*bg?p!kpe`F3@JC6mK3=FAYt^y;3mS;?1W(d|`0jkSn z1QARi0;Jzh1H>-@F%yd_HGmWYf+&#(a*H!_3vyE9i}Op1l2i4Ib25RXfl)k|p;u5@ Z!U0rclbfGXnv-e=GOHM53qK599DQgyzE$qtIUuSu36DHt6cC|Armbsm}vaV7)c&wj}PWK0gC)0NM t!Ez=46X-R+RHPx95JG9Dg)DJNEzp literal 0 HcmV?d00001 diff --git a/test/bytecode_pypy3.2/05_setattr.pyc b/test/bytecode_pypy3.2/05_setattr.pyc new file mode 100644 index 0000000000000000000000000000000000000000..09de469250c00fe3c8305ce5cbfe25afaae418b5 GIT binary patch literal 402 zcmb7AO-sW-5Ph4dwWaw7g5W7A*rJFR5%J^TrI#KGVq0puNN8eeW+N8TQ~eG8C;e5v z*(!Q<*m=yGnSF2BE~3`x_}o8(CMCx tzr_#d3sDIU)LdEwm@U&b5!$8~>nHsz{3`!Tewz|!c`>ard+t5d;tTdiNrwOc literal 0 HcmV?d00001 diff --git a/test/bytecode_pypy3.2/06_setif_comprehension.pyc b/test/bytecode_pypy3.2/06_setif_comprehension.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b708b74383fe985698ddcd8e7045e27791e77dc7 GIT binary patch literal 438 zcmbVHu}Z{15Ph4|^CDOXR@N3lAr~yP5fK#2Yb*p&mfU8%1hPrYY=ncm>ih!#O25k4 zMA6!T_jcIXH}7qNyk@QyrJ`~D;H-LZ-g-_sSqwo+QQ@D!)85@t*{Q@;leg0X|B80&~)A{PR~!K$(d$y z#j`fGZCc8y;-aqNX33$5FJ$HimwtKg5UNnk24DZ6H*8WfxE78q_rjm-{F~w5K_)(k Ot+TGAA&39)SbYI1`djD# literal 0 HcmV?d00001 diff --git a/test/simple_source/bug27+/05_setattr.py b/test/simple_source/bug27+/05_setattr.py new file mode 100644 index 00000000..6b40862b --- /dev/null +++ b/test/simple_source/bug27+/05_setattr.py @@ -0,0 +1,9 @@ +# Ensure PyPy handling of: +# key, value in slotstate.items(). +# PyPy uses LOOKUP_METHOD and CALL_METHOD instead +# of LOAD_ATTR and CALL_FUNCTION +def bug(state, slotstate): + if state: + if slotstate is not None: + for key, value in slotstate.items(): + setattr(state, key, 2) diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 03d17ee4..c47ae621 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -266,14 +266,12 @@ class Python2Parser(PythonParser): self.seen1024 = True rule = ('build_list ::= ' + 'expr1024 '*thousands + 'expr32 '*thirty32s + 'expr '*(v%32) + opname) - elif opname == 'CALL_METHOD': - # A PyPy speciality + elif opname == 'LOOKUP_METHOD': + # A PyPy speciality - DRY with parse3 self.add_unique_rule("load_attr ::= LOAD_FAST LOOKUP_METHOD", - opname_base, v, customize) + opname, v, customize) self.add_unique_rule("load_attr ::= LOAD_NAME LOOKUP_METHOD", - opname_base, v, customize) - self.add_unique_rule("call_function ::= expr CALL_METHOD", - opname_base, v, customize) + opname, v, customize) continue elif opname == 'JUMP_IF_NOT_DEBUG': self.add_unique_rule( @@ -313,12 +311,20 @@ class Python2Parser(PythonParser): rule = 'mkfunc ::= %s load_closure LOAD_CONST %s' % ('expr '*v, opname) # rule = 'mkfunc ::= %s closure_list LOAD_CONST %s' % ('expr '*v, opname) elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', - 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): - na = (v & 0xff) # positional parameters - nk = (v >> 8) & 0xff # keyword parameters + 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): + args_pos = (v & 0xff) # positional parameters + args_kw = (v >> 8) & 0xff # keyword parameters # number of apply equiv arguments: nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3 - rule = 'call_function ::= expr ' + 'expr '*na + 'kwarg '*nk \ + rule = 'call_function ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ + + 'expr ' * nak + opname + elif opname_base == 'CALL_METHOD': + # PyPy only - DRY with parse3 + args_pos = (v & 0xff) # positional parameters + args_kw = (v >> 8) & 0xff # keyword parameters + # number of apply equiv arguments: + nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 + rule = 'call_function ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ + 'expr ' * nak + opname else: raise Exception('unknown customize token %s' % opname) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 83ff1c66..95dc38e1 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -462,14 +462,12 @@ class Python3Parser(PythonParser): if opname_base == 'BUILD_TUPLE': rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname)) self.add_unique_rule(rule, opname, token.attr, customize) - elif opname == 'CALL_METHOD': - # A PyPy speciality + elif opname == 'LOOKUP_METHOD': + # A PyPy speciality - DRY with parse2 self.add_unique_rule("load_attr ::= LOAD_FAST LOOKUP_METHOD", opname, token.attr, customize) self.add_unique_rule("load_attr ::= LOAD_NAME LOOKUP_METHOD", opname, token.attr, customize) - self.add_unique_rule("call_function ::= expr CALL_METHOD", - opname, token.attr, customize) continue elif opname == 'JUMP_IF_NOT_DEBUG': self.add_unique_rule( @@ -526,6 +524,17 @@ class Python3Parser(PythonParser): rule = ('mkfunc ::= kwargs %sexpr %s' % ('pos_arg ' * args_pos, opname)) self.add_unique_rule(rule, opname, token.attr, customize) + elif opname_base == 'CALL_METHOD': + # PyPy only - DRY with parse2 + args_pos = (token.attr & 0xff) # positional parameters + args_kw = (token.attr >> 8) & 0xff # keyword parameters + # number of apply equiv arguments: + nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 + rule = ('call_function ::= expr ' + + ('pos_arg ' * args_pos) + + ('kwarg ' * args_kw) + + 'expr ' * nak + token.type) + self.add_unique_rule(rule, opname, token.attr, customize) elif opname.startswith('MAKE_CLOSURE'): # DRY with MAKE_FUNCTION # Note: this probably doesn't handle kwargs proprerly diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index ff4b0bcb..2faca988 100755 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -192,8 +192,11 @@ class Scanner2(scan.Scanner): opname = '%s_%d' % (opname, oparg) if op != self.opc.BUILD_SLICE: customize[opname] = oparg - elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'): - customize[opname] = oparg + elif self.is_pypy and opname in ('LOOKUP_METHOD', 'JUMP_IF_NOT_DEBUG'): + # The value in the dict is used in rule uniquness key. + # These ops need only be done once. Hence we use arbitrary constant + # 0. + customize[opname] = 0 elif op == self.opc.JUMP_ABSOLUTE: target = self.get_target(offset) if target < offset: diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index af1320b4..d1b9d84e 100755 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -52,7 +52,7 @@ class Scanner27(Scanner2): # opcodes with expect a variable number pushed values whose # count is in the opcode. For parsing we generally change the # opcode name to include that number. - self.varargs_ops = frozenset([ + varargs_ops = set([ self.opc.BUILD_LIST, self.opc.BUILD_TUPLE, self.opc.BUILD_SLICE, self.opc.UNPACK_SEQUENCE, self.opc.MAKE_FUNCTION, self.opc.CALL_FUNCTION, @@ -62,6 +62,10 @@ class Scanner27(Scanner2): # New in Python 2.7 self.opc.BUILD_SET, self.opc.BUILD_MAP]) + if is_pypy: + varargs_ops.add(self.opc.CALL_METHOD) + self.varargs_ops = frozenset(varargs_ops) + # "setup" opcodes self.setup_ops = frozenset([ self.opc.SETUP_EXCEPT, self.opc.SETUP_FINALLY, diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index bdeb84e0..27ba57ca 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -89,13 +89,15 @@ class Scanner3(scan.Scanner): # Opcodes that take a variable number of arguments # (expr's) - self.varargs = frozenset([self.opc.BUILD_LIST, - self.opc.BUILD_TUPLE, - self.opc.BUILD_SET, - self.opc.BUILD_SLICE, - self.opc.BUILD_MAP, - self.opc.UNPACK_SEQUENCE, - self.opc.RAISE_VARARGS]) + varargs_ops = set([ + self.opc.BUILD_LIST, self.opc.BUILD_TUPLE, + self.opc.BUILD_SET, self.opc.BUILD_SLICE, + self.opc.BUILD_MAP, self.opc.UNPACK_SEQUENCE, + self.opc.RAISE_VARARGS]) + + if is_pypy: + varargs_ops.add(self.opc.CALL_METHOD) + self.varargs_ops = frozenset(varargs_ops) # Not really a set, but still clasification-like self.statement_opcode_sequences = [ @@ -234,7 +236,7 @@ class Scanner3(scan.Scanner): ) ) continue - elif op in self.varargs: + elif op in self.varargs_ops: pos_args = inst.argval opname = '%s_%d' % (opname, pos_args) elif self.is_pypy and opname in ('CALL_METHOD', 'JUMP_IF_NOT_DEBUG'):