From 31ebe88b389e9153779806da77431b3b96a2faf8 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 2 Jan 2016 22:59:02 -0500 Subject: [PATCH] Start to DRY opcode code. Limited support for decopyling Python 3.5 --- uncompyle6/opcodes/opcode_23.py | 2 + uncompyle6/opcodes/opcode_25.py | 2 + uncompyle6/opcodes/opcode_26.py | 2 + uncompyle6/opcodes/opcode_27.py | 2 + uncompyle6/opcodes/opcode_32.py | 200 ++----------------- uncompyle6/opcodes/opcode_33.py | 185 ++--------------- uncompyle6/opcodes/opcode_34.py | 209 +++---------------- uncompyle6/opcodes/opcode_35.py | 81 ++++++++ uncompyle6/opcodes/opcode_3x.py | 210 +++++++++++++++++++ uncompyle6/parser.py | 8 +- uncompyle6/scanner.py | 8 +- uncompyle6/scanners/scanner32.py | 4 +- uncompyle6/scanners/scanner33.py | 4 +- uncompyle6/scanners/scanner34.py | 4 +- uncompyle6/scanners/scanner35.py | 333 +++++++++++++++++++++++++++++++ 15 files changed, 708 insertions(+), 546 deletions(-) create mode 100644 uncompyle6/opcodes/opcode_35.py create mode 100644 uncompyle6/opcodes/opcode_3x.py create mode 100644 uncompyle6/scanners/scanner35.py diff --git a/uncompyle6/opcodes/opcode_23.py b/uncompyle6/opcodes/opcode_23.py index 90779e99..510fc51f 100755 --- a/uncompyle6/opcodes/opcode_23.py +++ b/uncompyle6/opcodes/opcode_23.py @@ -7,6 +7,8 @@ This is a superset of Python 2.3's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ +# FIXME: DRY this along the lines of opcode_3x. + cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', 'is not', 'exception match', 'BAD') diff --git a/uncompyle6/opcodes/opcode_25.py b/uncompyle6/opcodes/opcode_25.py index af71ac51..a27a8565 100755 --- a/uncompyle6/opcodes/opcode_25.py +++ b/uncompyle6/opcodes/opcode_25.py @@ -7,6 +7,8 @@ This is a superset of Python 2.5's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ +# FIXME: DRY this along the lines of opcode_3x. + cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', 'is not', 'exception match', 'BAD') diff --git a/uncompyle6/opcodes/opcode_26.py b/uncompyle6/opcodes/opcode_26.py index 949d84c3..51d918d9 100755 --- a/uncompyle6/opcodes/opcode_26.py +++ b/uncompyle6/opcodes/opcode_26.py @@ -7,6 +7,8 @@ This is a superset of Python 3.4's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ +# FIXME: DRY this along the lines of opcode_3x. + cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', 'is not', 'exception match', 'BAD') diff --git a/uncompyle6/opcodes/opcode_27.py b/uncompyle6/opcodes/opcode_27.py index 92e8559e..0566bf17 100755 --- a/uncompyle6/opcodes/opcode_27.py +++ b/uncompyle6/opcodes/opcode_27.py @@ -7,6 +7,8 @@ This is a superset of Python 3.4's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ +# FIXME: DRY this along the lines of opcode_3x. + cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', 'is not', 'exception match', 'BAD') diff --git a/uncompyle6/opcodes/opcode_32.py b/uncompyle6/opcodes/opcode_32.py index ab9e83b8..90fe7d5b 100644 --- a/uncompyle6/opcodes/opcode_32.py +++ b/uncompyle6/opcodes/opcode_32.py @@ -3,42 +3,27 @@ CPython 3.2 bytecode opcodes This is used in scanner (bytecode disassembly) and parser (Python grammar). -This is a superset of Python 3.4's opcode.py with some opcodes that simplify +This is a superset of Python 3.2's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ -cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', - 'is not', 'exception match', 'BAD') +from copy import deepcopy -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasnargs = [] +import uncompyle6.opcodes.opcode_3x as opcode_3x +from uncompyle6.opcodes.opcode_3x import fields2copy + +# FIXME: can we DRY this even more? opmap = {} opname = [''] * 256 -for op in range(256): opname[op] = '<%r>' % (op,) -del op +hasjrel = [] +hasjabs = [] -def def_op(name, op): - opname[op] = name - opmap[name] = op +for object in fields2copy: + globals()[object] = deepcopy(getattr(opcode_3x, object)) -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) +# There are no opcodes to add or change. +# If there were, they'd be listed below. def updateGlobal(): # JUMP_OPs are used in verification are set in the scanner @@ -50,165 +35,14 @@ def updateGlobal(): globals().update(dict([(k.replace('+', '_'), v) for (k, v) in opmap.items()])) globals().update({'JUMP_OPs': map(lambda op: opname[op], hasjrel + hasjabs)}) -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('STOP_CODE', 0) -def_op('POP_TOP', 1) -def_op('ROT_TWO', 2) -def_op('ROT_THREE', 3) -def_op('DUP_TOP', 4) -def_op('DUP_TOP_TWO', 5) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_POWER', 19) -def_op('BINARY_MULTIPLY', 20) - -def_op('BINARY_MODULO', 22) -def_op('BINARY_ADD', 23) -def_op('BINARY_SUBTRACT', 24) -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_FLOOR_DIVIDE', 26) -def_op('BINARY_TRUE_DIVIDE', 27) -def_op('INPLACE_FLOOR_DIVIDE', 28) -def_op('INPLACE_TRUE_DIVIDE', 29) - -# Gone from Python 3 are Python2's -# SLICE+0 .. SLICE+3 -# STORE_SLICE+0 .. STORE_SLICE+3 -# DELETE_SLICE+0 .. DELETE_SLICE+3 - -def_op('STORE_MAP', 54) -def_op('INPLACE_ADD', 55) -def_op('INPLACE_SUBTRACT', 56) -def_op('INPLACE_MULTIPLY', 57) - -def_op('INPLACE_MODULO', 59) -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) -def_op('BINARY_LSHIFT', 62) -def_op('BINARY_RSHIFT', 63) -def_op('BINARY_AND', 64) -def_op('BINARY_XOR', 65) -def_op('BINARY_OR', 66) -def_op('INPLACE_POWER', 67) -def_op('GET_ITER', 68) -def_op('STORE_LOCALS', 69) - -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) - -# Python3 drops/changes: -# def_op('PRINT_ITEM', 71) -# def_op('PRINT_NEWLINE', 72) -# def_op('PRINT_ITEM_TO', 73) -# def_op('PRINT_NEWLINE_TO', 74) - -def_op('INPLACE_LSHIFT', 75) -def_op('INPLACE_RSHIFT', 76) -def_op('INPLACE_AND', 77) -def_op('INPLACE_XOR', 78) -def_op('INPLACE_OR', 79) -def_op('BREAK_LOOP', 80) -def_op('WITH_CLEANUP', 81) - -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) - -def_op('YIELD_VALUE', 86) -def_op('POP_BLOCK', 87) -def_op('END_FINALLY', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # Opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" - -# Python 2's DUP_TOPX is gone - -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries (upto 255) -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list - -jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip -jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code -jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jabs_op('JUMP_ABSOLUTE', 113) # "" -jabs_op('POP_JUMP_IF_FALSE', 114) # "" -jabs_op('POP_JUMP_IF_TRUE', 115) # "" - -name_op('LOAD_GLOBAL', 116) # Index in name list - -jabs_op('CONTINUE_LOOP', 119) # Target address -jrel_op('SETUP_LOOP', 120) # Distance to target address -jrel_op('SETUP_EXCEPT', 121) # "" -jrel_op('SETUP_FINALLY', 122) # "" - -def_op('LOAD_FAST', 124) # Local variable number -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) - -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) -hasnargs.append(131) -def_op('MAKE_FUNCTION', 132) # Number of args with default values -def_op('BUILD_SLICE', 133) # Number of items -def_op('MAKE_CLOSURE', 134) -def_op('LOAD_CLOSURE', 135) -hasfree.append(135) -def_op('LOAD_DEREF', 136) -hasfree.append(136) -def_op('STORE_DEREF', 137) -hasfree.append(137) -def_op('DELETE_DEREF', 138) -hasfree.append(138) - -def_op('CALL_FUNCTION_VAR', 140) # #args + (#kwargs << 8) -hasnargs.append(140) -def_op('CALL_FUNCTION_KW', 141) # #args + (#kwargs << 8) -hasnargs.append(141) -def_op('CALL_FUNCTION_VAR_KW', 142) # #args + (#kwargs << 8) -hasnargs.append(142) - -jrel_op('SETUP_WITH', 143) - -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 - updateGlobal() -del def_op, name_op, jrel_op, jabs_op from uncompyle6 import PYTHON_VERSION if PYTHON_VERSION == 3.2: import dis + # for item in dis.opmap.items(): + # if item not in opmap.items(): + # print(item) assert all(item in opmap.items() for item in dis.opmap.items()) + +# opcode_3x.dump_opcodes(opmap) diff --git a/uncompyle6/opcodes/opcode_33.py b/uncompyle6/opcodes/opcode_33.py index 392067b0..69e5da64 100644 --- a/uncompyle6/opcodes/opcode_33.py +++ b/uncompyle6/opcodes/opcode_33.py @@ -7,37 +7,31 @@ This is a superset of Python 3.3's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ -cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', - 'is not', 'exception match', 'BAD') +from copy import deepcopy -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] +import uncompyle6.opcodes.opcode_3x as opcode_3x +from uncompyle6.opcodes.opcode_3x import fields2copy, rm_op + +# FIXME: can we DRY this even more? opmap = {} opname = [''] * 256 -for op in range(256): opname[op] = '<%r>' % (op,) -del op +hasconst = [] +hasjrel = [] +hasjabs = [] def def_op(name, op): opname[op] = name opmap[name] = op -def name_op(name, op): - def_op(name, op) - hasname.append(op) +for object in fields2copy: + globals()[object] = deepcopy(getattr(opcode_3x, object)) -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) +# Below are opcodes since Python 3.2 -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) +rm_op(opname, opmap, 'STOP_CODE', 0) + +def_op('YIELD_FROM', 72) def updateGlobal(): # JUMP_OPs are used in verification are set in the scanner @@ -49,156 +43,9 @@ def updateGlobal(): globals().update(dict([(k.replace('+', '_'), v) for (k, v) in opmap.items()])) globals().update({'JUMP_OPs': map(lambda op: opname[op], hasjrel + hasjabs)}) -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('STOP_CODE', 0) -def_op('POP_TOP', 1) -def_op('ROT_TWO', 2) -def_op('ROT_THREE', 3) -def_op('DUP_TOP', 4) -def_op('DUP_TOP_TWO', 5) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_POWER', 19) -def_op('BINARY_MULTIPLY', 20) - -def_op('BINARY_MODULO', 22) -def_op('BINARY_ADD', 23) -def_op('BINARY_SUBTRACT', 24) -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_FLOOR_DIVIDE', 26) -def_op('BINARY_TRUE_DIVIDE', 27) -def_op('INPLACE_FLOOR_DIVIDE', 28) -def_op('INPLACE_TRUE_DIVIDE', 29) - -# Gone from Python 3 are -# Python 2's SLICE+0 .. SLICE+3 - -def_op('STORE_MAP', 54) -def_op('INPLACE_ADD', 55) -def_op('INPLACE_SUBTRACT', 56) -def_op('INPLACE_MULTIPLY', 57) - -def_op('INPLACE_MODULO', 59) -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) -def_op('BINARY_LSHIFT', 62) -def_op('BINARY_RSHIFT', 63) -def_op('BINARY_AND', 64) -def_op('BINARY_XOR', 65) -def_op('BINARY_OR', 66) -def_op('INPLACE_POWER', 67) -def_op('GET_ITER', 68) -def_op('STORE_LOCALS', 69) - -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) - -# Python3 drops/changes: -# def_op('PRINT_ITEM', 71) -# def_op('PRINT_NEWLINE', 72) -def_op('YIELD_FROM', 72) -# def_op('PRINT_ITEM_TO', 73) -# def_op('PRINT_NEWLINE_TO', 74) - -def_op('INPLACE_LSHIFT', 75) -def_op('INPLACE_RSHIFT', 76) -def_op('INPLACE_AND', 77) -def_op('INPLACE_XOR', 78) -def_op('INPLACE_OR', 79) -def_op('BREAK_LOOP', 80) -def_op('WITH_CLEANUP', 81) - -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) - -def_op('YIELD_VALUE', 86) -def_op('POP_BLOCK', 87) -def_op('END_FINALLY', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # Opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries (upto 255) -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list - -jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip -jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code -jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jabs_op('JUMP_ABSOLUTE', 113) # "" -jabs_op('POP_JUMP_IF_FALSE', 114) # "" -jabs_op('POP_JUMP_IF_TRUE', 115) # "" - -name_op('LOAD_GLOBAL', 116) # Index in name list - -jabs_op('CONTINUE_LOOP', 119) # Target address -jrel_op('SETUP_LOOP', 120) # Distance to target address -jrel_op('SETUP_EXCEPT', 121) # "" -jrel_op('SETUP_FINALLY', 122) # "" - -def_op('LOAD_FAST', 124) # Local variable number -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) - -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) -def_op('MAKE_FUNCTION', 132) # Number of args with default values -def_op('BUILD_SLICE', 133) # Number of items -def_op('MAKE_CLOSURE', 134) -def_op('LOAD_CLOSURE', 135) -hasfree.append(135) -def_op('LOAD_DEREF', 136) -hasfree.append(136) -def_op('STORE_DEREF', 137) -hasfree.append(137) -def_op('DELETE_DEREF', 138) -hasfree.append(138) - -def_op('CALL_FUNCTION_VAR', 140) # #args + (#kwargs << 8) -def_op('CALL_FUNCTION_KW', 141) # #args + (#kwargs << 8) -def_op('CALL_FUNCTION_VAR_KW', 142) # #args + (#kwargs << 8) - -jrel_op('SETUP_WITH', 143) - -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 - updateGlobal() -del def_op, name_op, jrel_op, jabs_op +# FIXME: turn into pytest test from uncompyle6 import PYTHON_VERSION if PYTHON_VERSION == 3.3: import dis @@ -206,3 +53,5 @@ if PYTHON_VERSION == 3.3: # if item not in opmap.items(): # print(item) assert all(item in opmap.items() for item in dis.opmap.items()) + +# opcode3x.dump_opcodes(opmap) diff --git a/uncompyle6/opcodes/opcode_34.py b/uncompyle6/opcodes/opcode_34.py index c5b8eb2e..8565a126 100644 --- a/uncompyle6/opcodes/opcode_34.py +++ b/uncompyle6/opcodes/opcode_34.py @@ -1,44 +1,41 @@ """ CPython 3.4 bytecode opcodes -This is used in scanner (bytecode disassembly) and parser (Python grammar). +used in scanner (bytecode disassembly) and parser (Python grammar) This is a superset of Python 3.4's opcode.py with some opcodes that simplify parsing and semantic interpretation. """ -cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', - 'is not', 'exception match', 'BAD') +from copy import deepcopy -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasnargs = [] +import uncompyle6.opcodes.opcode_3x as opcode_3x +from uncompyle6.opcodes.opcode_3x import fields2copy, hasfree, rm_op + +# FIXME: can we DRY this even more? opmap = {} opname = [''] * 256 -for op in range(256): opname[op] = '<%r>' % (op,) -del op +hasconst = [] +hasjrel = [] +hasjabs = [] def def_op(name, op): opname[op] = name opmap[name] = op -def name_op(name, op): - def_op(name, op) - hasname.append(op) +for object in fields2copy: + globals()[object] = deepcopy(getattr(opcode_3x, object)) -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) +# Below are opcodes changes since Python 3.2 -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) +rm_op(opname, opmap, 'STOP_CODE', 0) +rm_op(opname, opmap, 'STORE_LOCALS', 69) + +# These are new since Python 3.3 +def_op('YIELD_FROM', 72) +def_op('LOAD_CLASSDEREF', 148) +hasfree.append(148) def updateGlobal(): # JUMP_OPs are used in verification are set in the scanner @@ -50,171 +47,15 @@ def updateGlobal(): globals().update(dict([(k.replace('+', '_'), v) for (k, v) in opmap.items()])) globals().update({'JUMP_OPs': map(lambda op: opname[op], hasjrel + hasjabs)}) -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('STOP_CODE', 0) -def_op('POP_TOP', 1) -def_op('ROT_TWO', 2) -def_op('ROT_THREE', 3) -def_op('DUP_TOP', 4) -def_op('DUP_TOP_TWO', 5) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_POWER', 19) -def_op('BINARY_MULTIPLY', 20) - -def_op('BINARY_MODULO', 22) -def_op('BINARY_ADD', 23) -def_op('BINARY_SUBTRACT', 24) -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_FLOOR_DIVIDE', 26) -def_op('BINARY_TRUE_DIVIDE', 27) -def_op('INPLACE_FLOOR_DIVIDE', 28) -def_op('INPLACE_TRUE_DIVIDE', 29) - -# Gone from Python 3 are Python2's -# SLICE+0 .. SLICE+3 -# STORE_SLICE+0 .. STORE_SLICE+3 -# DELETE_SLICE+0 .. DELETE_SLICE+3 - -def_op('STORE_MAP', 54) -def_op('INPLACE_ADD', 55) -def_op('INPLACE_SUBTRACT', 56) -def_op('INPLACE_MULTIPLY', 57) - -def_op('INPLACE_MODULO', 59) -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) -def_op('BINARY_LSHIFT', 62) -def_op('BINARY_RSHIFT', 63) -def_op('BINARY_AND', 64) -def_op('BINARY_XOR', 65) -def_op('BINARY_OR', 66) -def_op('INPLACE_POWER', 67) -def_op('GET_ITER', 68) - -# FIXME: no code generates this -def_op('STORE_LOCALS', 69) - -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) -def_op('YIELD_FROM', 72) - -# Python3 drops/changes: -# def_op('PRINT_ITEM', 71) -# def_op('PRINT_NEWLINE', 72) -# def_op('PRINT_ITEM_TO', 73) -# def_op('PRINT_NEWLINE_TO', 74) - -def_op('INPLACE_LSHIFT', 75) -def_op('INPLACE_RSHIFT', 76) -def_op('INPLACE_AND', 77) -def_op('INPLACE_XOR', 78) -def_op('INPLACE_OR', 79) -def_op('BREAK_LOOP', 80) -def_op('WITH_CLEANUP', 81) - -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) - -def_op('YIELD_VALUE', 86) -def_op('POP_BLOCK', 87) -def_op('END_FINALLY', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # Opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" - -# Python 2's DUP_TOPX is gone - -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries (upto 255) -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list - -jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip -jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code -jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jabs_op('JUMP_ABSOLUTE', 113) # "" -jabs_op('POP_JUMP_IF_FALSE', 114) # "" -jabs_op('POP_JUMP_IF_TRUE', 115) # "" - -name_op('LOAD_GLOBAL', 116) # Index in name list - -jabs_op('CONTINUE_LOOP', 119) # Target address -jrel_op('SETUP_LOOP', 120) # Distance to target address -jrel_op('SETUP_EXCEPT', 121) # "" -jrel_op('SETUP_FINALLY', 122) # "" - -def_op('LOAD_FAST', 124) # Local variable number -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) - -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) -hasnargs.append(131) -def_op('MAKE_FUNCTION', 132) # Number of args with default values -def_op('BUILD_SLICE', 133) # Number of items -def_op('MAKE_CLOSURE', 134) -def_op('LOAD_CLOSURE', 135) -hasfree.append(135) -def_op('LOAD_DEREF', 136) -hasfree.append(136) -def_op('STORE_DEREF', 137) -hasfree.append(137) -def_op('DELETE_DEREF', 138) -hasfree.append(138) - -def_op('CALL_FUNCTION_VAR', 140) # #args + (#kwargs << 8) -hasnargs.append(140) -def_op('CALL_FUNCTION_KW', 141) # #args + (#kwargs << 8) -hasnargs.append(141) -def_op('CALL_FUNCTION_VAR_KW', 142) # #args + (#kwargs << 8) -hasnargs.append(142) - -jrel_op('SETUP_WITH', 143) - -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) - -def_op('LOAD_CLASSDEREF', 148) -hasfree.append(148) - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 - updateGlobal() -del def_op, name_op, jrel_op, jabs_op +# FIXME: turn into pytest test from uncompyle6 import PYTHON_VERSION if PYTHON_VERSION == 3.4: import dis + # for item in dis.opmap.items(): + # if item not in opmap.items(): + # print(item) assert all(item in opmap.items() for item in dis.opmap.items()) + +# opcode_3x.dump_opcodes(opmap) diff --git a/uncompyle6/opcodes/opcode_35.py b/uncompyle6/opcodes/opcode_35.py new file mode 100644 index 00000000..e1d09772 --- /dev/null +++ b/uncompyle6/opcodes/opcode_35.py @@ -0,0 +1,81 @@ +# (C) Copyright 2016 by Rocky Bernstein +""" +CPython 3.5 bytecode opcodes + +used in scanner (bytecode disassembly) and parser (Python grammar) + +This is a superset of Python 3.5's opcode.py with some opcodes that simplify +parsing and semantic interpretation. +""" + +from copy import deepcopy + +import uncompyle6.opcodes.opcode_3x as opcode_3x +from uncompyle6.opcodes.opcode_3x import fields2copy, hasfree, rm_op + +# FIXME: can we DRY this even more? + +opmap = {} +opname = [''] * 256 +hasconst = [] +hasjrel = [] +hasjabs = [] + +def def_op(name, op): + opname[op] = name + opmap[name] = op + +for object in fields2copy: + globals()[object] = deepcopy(getattr(opcode_3x, object)) + +# Below are opcodes changes since Python 3.2 + +rm_op(opname, opmap, 'STOP_CODE', 0) +rm_op(opname, opmap, 'STORE_LOCALS', 69) + +# These are new since Python 3.3 +def_op('YIELD_FROM', 72) +def_op('LOAD_CLASSDEREF', 148) + +# These are new since Python 3.4 +def_op('BINARY_MATRIX_MULTIPLY', 16) +def_op('INPLACE_MATRIX_MULTIPLY', 17) +def_op('GET_AITER', 50) +def_op('GET_ANEXT', 51) +def_op('BEFORE_ASYNC_WITH', 52) +def_op('GET_YIELD_FROM_ITER', 69) +def_op('GET_AWAITABLE', 73) +def_op('WITH_CLEANUP_START', 81) +def_op('WITH_CLEANUP_FINISH', 82) +def_op('BUILD_LIST_UNPACK', 149) +def_op('BUILD_MAP_UNPACK', 150) +def_op('BUILD_MAP_UNPACK_WITH_CALL', 151) +def_op('BUILD_TUPLE_UNPACK', 152) +def_op('BUILD_SET_UNPACK', 153) +def_op('SETUP_ASYNC_WITH', 154) + + +hasfree.append(148) + +def updateGlobal(): + # JUMP_OPs are used in verification are set in the scanner + # and used in the parser grammar + globals().update({'PJIF': opmap['POP_JUMP_IF_FALSE']}) + globals().update({'PJIT': opmap['POP_JUMP_IF_TRUE']}) + globals().update({'JA': opmap['JUMP_ABSOLUTE']}) + globals().update({'JF': opmap['JUMP_FORWARD']}) + globals().update(dict([(k.replace('+', '_'), v) for (k, v) in opmap.items()])) + globals().update({'JUMP_OPs': map(lambda op: opname[op], hasjrel + hasjabs)}) + +updateGlobal() + +# FIXME: turn into pytest test +from uncompyle6 import PYTHON_VERSION +if PYTHON_VERSION == 3.5: + import dis + for item in dis.opmap.items(): + if item not in opmap.items(): + print(item) + assert all(item in opmap.items() for item in dis.opmap.items()) + +# opcode_3x.dump_opcodes(opmap) diff --git a/uncompyle6/opcodes/opcode_3x.py b/uncompyle6/opcodes/opcode_3x.py new file mode 100644 index 00000000..d4b4a71f --- /dev/null +++ b/uncompyle6/opcodes/opcode_3x.py @@ -0,0 +1,210 @@ +""" +CPython 3.2 bytecode opcodes to be used as a base for other opcodes including 3.2. +If this file changes the other opcode files may have to a adjusted accordingly. +""" + +cmp_op = ('<', '<=', '==', '!=', '>', '>=', 'in', 'not in', 'is', + 'is not', 'exception match', 'BAD') + +hasconst = [] +hasname = [] +hasjrel = [] +hasjabs = [] +haslocal = [] +hascompare = [] +hasfree = [] +hasnargs = [] + +opmap = {} +opname = [''] * 256 +for op in range(256): opname[op] = '<%r>' % (op,) +del op + +def def_op(name, op): + opname[op] = name + opmap[name] = op + +def name_op(name, op): + def_op(name, op) + hasname.append(op) + +def jrel_op(name, op): + def_op(name, op) + hasjrel.append(op) + +def jabs_op(name, op): + def_op(name, op) + hasjabs.append(op) + +# Instruction opcodes for compiled code +# Blank lines correspond to available opcodes + +def_op('STOP_CODE', 0) +def_op('POP_TOP', 1) +def_op('ROT_TWO', 2) +def_op('ROT_THREE', 3) +def_op('DUP_TOP', 4) +def_op('DUP_TOP_TWO', 5) + +def_op('NOP', 9) +def_op('UNARY_POSITIVE', 10) +def_op('UNARY_NEGATIVE', 11) +def_op('UNARY_NOT', 12) + +def_op('UNARY_INVERT', 15) + +def_op('BINARY_POWER', 19) +def_op('BINARY_MULTIPLY', 20) + +def_op('BINARY_MODULO', 22) +def_op('BINARY_ADD', 23) +def_op('BINARY_SUBTRACT', 24) +def_op('BINARY_SUBSCR', 25) +def_op('BINARY_FLOOR_DIVIDE', 26) +def_op('BINARY_TRUE_DIVIDE', 27) +def_op('INPLACE_FLOOR_DIVIDE', 28) +def_op('INPLACE_TRUE_DIVIDE', 29) + +# Gone from Python 3 are Python2's +# SLICE+0 .. SLICE+3 +# STORE_SLICE+0 .. STORE_SLICE+3 +# DELETE_SLICE+0 .. DELETE_SLICE+3 + +def_op('STORE_MAP', 54) +def_op('INPLACE_ADD', 55) +def_op('INPLACE_SUBTRACT', 56) +def_op('INPLACE_MULTIPLY', 57) + +def_op('INPLACE_MODULO', 59) +def_op('STORE_SUBSCR', 60) +def_op('DELETE_SUBSCR', 61) +def_op('BINARY_LSHIFT', 62) +def_op('BINARY_RSHIFT', 63) +def_op('BINARY_AND', 64) +def_op('BINARY_XOR', 65) +def_op('BINARY_OR', 66) +def_op('INPLACE_POWER', 67) +def_op('GET_ITER', 68) +def_op('STORE_LOCALS', 69) + +def_op('PRINT_EXPR', 70) +def_op('LOAD_BUILD_CLASS', 71) + +# Python3 drops/changes: +# def_op('PRINT_ITEM', 71) +# def_op('PRINT_NEWLINE', 72) +# def_op('PRINT_ITEM_TO', 73) +# def_op('PRINT_NEWLINE_TO', 74) + +def_op('INPLACE_LSHIFT', 75) +def_op('INPLACE_RSHIFT', 76) +def_op('INPLACE_AND', 77) +def_op('INPLACE_XOR', 78) +def_op('INPLACE_OR', 79) +def_op('BREAK_LOOP', 80) +def_op('WITH_CLEANUP', 81) + +def_op('RETURN_VALUE', 83) +def_op('IMPORT_STAR', 84) + +def_op('YIELD_VALUE', 86) +def_op('POP_BLOCK', 87) +def_op('END_FINALLY', 88) +def_op('POP_EXCEPT', 89) + +HAVE_ARGUMENT = 90 # Opcodes from here have an argument: + +name_op('STORE_NAME', 90) # Index in name list +name_op('DELETE_NAME', 91) # "" +def_op('UNPACK_SEQUENCE', 92) # Number of tuple items +jrel_op('FOR_ITER', 93) +def_op('UNPACK_EX', 94) +name_op('STORE_ATTR', 95) # Index in name list +name_op('DELETE_ATTR', 96) # "" +name_op('STORE_GLOBAL', 97) # "" +name_op('DELETE_GLOBAL', 98) # "" + +# Python 2's DUP_TOPX is gone + +def_op('LOAD_CONST', 100) # Index in const list +hasconst.append(100) +name_op('LOAD_NAME', 101) # Index in name list +def_op('BUILD_TUPLE', 102) # Number of tuple items +def_op('BUILD_LIST', 103) # Number of list items +def_op('BUILD_SET', 104) # Number of set items +def_op('BUILD_MAP', 105) # Number of dict entries (upto 255) +name_op('LOAD_ATTR', 106) # Index in name list +def_op('COMPARE_OP', 107) # Comparison operator +hascompare.append(107) +name_op('IMPORT_NAME', 108) # Index in name list +name_op('IMPORT_FROM', 109) # Index in name list + +jrel_op('JUMP_FORWARD', 110) # Number of bytes to skip +jabs_op('JUMP_IF_FALSE_OR_POP', 111) # Target byte offset from beginning of code +jabs_op('JUMP_IF_TRUE_OR_POP', 112) # "" +jabs_op('JUMP_ABSOLUTE', 113) # "" +jabs_op('POP_JUMP_IF_FALSE', 114) # "" +jabs_op('POP_JUMP_IF_TRUE', 115) # "" + +name_op('LOAD_GLOBAL', 116) # Index in name list + +jabs_op('CONTINUE_LOOP', 119) # Target address +jrel_op('SETUP_LOOP', 120) # Distance to target address +jrel_op('SETUP_EXCEPT', 121) # "" +jrel_op('SETUP_FINALLY', 122) # "" + +def_op('LOAD_FAST', 124) # Local variable number +haslocal.append(124) +def_op('STORE_FAST', 125) # Local variable number +haslocal.append(125) +def_op('DELETE_FAST', 126) # Local variable number +haslocal.append(126) + +def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) +def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) +hasnargs.append(131) +def_op('MAKE_FUNCTION', 132) # Number of args with default values +def_op('BUILD_SLICE', 133) # Number of items +def_op('MAKE_CLOSURE', 134) +def_op('LOAD_CLOSURE', 135) +hasfree.append(135) +def_op('LOAD_DEREF', 136) +hasfree.append(136) +def_op('STORE_DEREF', 137) +hasfree.append(137) +def_op('DELETE_DEREF', 138) +hasfree.append(138) + +def_op('CALL_FUNCTION_VAR', 140) # #args + (#kwargs << 8) +hasnargs.append(140) +def_op('CALL_FUNCTION_KW', 141) # #args + (#kwargs << 8) +hasnargs.append(141) +def_op('CALL_FUNCTION_VAR_KW', 142) # #args + (#kwargs << 8) +hasnargs.append(142) + +jrel_op('SETUP_WITH', 143) + +def_op('LIST_APPEND', 145) +def_op('SET_ADD', 146) +def_op('MAP_ADD', 147) + +def_op('EXTENDED_ARG', 144) +EXTENDED_ARG = 144 + +fields2copy = """cmp_op hasconst hasname hasjrel hasjabs haslocal hascompare hasfree hasnargs +opmap opname HAVE_ARGUMENT EXTENDED_ARG""".split() + +def rm_op(opname, opmap, name, op): + # opname is an array, so we need to keep the position in there. + opname[op] = '' + + assert opmap[name] == op + del opmap[name] + +def dump_opcodes(opmap): + """Utility for dumping opcodes""" + op2name = {} + for k in opmap.keys(): + op2name[opmap[k]] = k + for i in sorted(op2name.keys()): + print("%-3s %s" % (str(i), op2name[i])) diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 8a1d56fb..2951bf70 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -1,7 +1,7 @@ -# Copyright (c) 1999 John Aycock -# Copyright (c) 2000-2002 by hartmut Goebel +# Copyright (c) 2015-2016 Rocky Bernstein # Copyright (c) 2005 by Dan Pascu -# Copyright (c) 2015 Rocky Bernstein +# Copyright (c) 2000-2002 by hartmut Goebel +# Copyright (c) 1999 John Aycock """ Common spark parser routines Python. """ @@ -105,7 +105,7 @@ def python_parser(version, co, out=sys.stdout, showasm=False, if __name__ == '__main__': def parse_test(co): - from uncompyl6e import PYTHON_VERSION + from uncompyl6 import PYTHON_VERSION ast = python_parser(PYTHON_VERSION, co, showasm=True) print(ast) return diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 7fc3e3cc..c8cd3038 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -29,7 +29,8 @@ if PYTHON3: else: L65536 = long(65536) # NOQA -from uncompyle6.opcodes import opcode_25, opcode_26, opcode_27, opcode_32, opcode_33, opcode_34 +from uncompyle6.opcodes import (opcode_25, opcode_26, opcode_27, + opcode_32, opcode_33, opcode_34, opcode_35) class Code: @@ -61,6 +62,8 @@ class Scanner(object): self.opc = opcode_33 elif version == 3.4: self.opc = opcode_34 + elif version == 3.5: + self.opc = opcode_35 else: raise TypeError("%s is not a Python version I know about" % version) @@ -305,6 +308,9 @@ def get_scanner(version): elif version == 3.4: import uncompyle6.scanners.scanner34 as scan scanner = scan.Scanner34(version) + elif version == 3.5: + import uncompyle6.scanners.scanner35 as scan + scanner = scan.Scanner35(version) else: raise RuntimeError("Unsupported Python version %d" % version) return scanner diff --git a/uncompyle6/scanners/scanner32.py b/uncompyle6/scanners/scanner32.py index b7df4969..39a20287 100644 --- a/uncompyle6/scanners/scanner32.py +++ b/uncompyle6/scanners/scanner32.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015 by Rocky Bernstein +# Copyright (c) 2015-2016 by Rocky Bernstein """ Python 3.2 bytecode scanner/deparser @@ -23,7 +23,7 @@ class Scanner32(scan3.Scanner3): if __name__ == "__main__": import inspect co = inspect.currentframe().f_code - tokens, customize = Scanner32().disassemble(co) + tokens, customize = Scanner32(3.2).disassemble(co) for t in tokens: print(t) pass diff --git a/uncompyle6/scanners/scanner33.py b/uncompyle6/scanners/scanner33.py index 67cd5912..d78427c9 100644 --- a/uncompyle6/scanners/scanner33.py +++ b/uncompyle6/scanners/scanner33.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015 by Rocky Bernstein +# Copyright (c) 2015-2016 by Rocky Bernstein """ Python 3 bytecode scanner/deparser @@ -23,7 +23,7 @@ class Scanner33(scan3.Scanner3): if __name__ == "__main__": import inspect co = inspect.currentframe().f_code - tokens, customize = Scanner33().disassemble(co) + tokens, customize = Scanner33(3.3).disassemble(co) for t in tokens: print(t) pass diff --git a/uncompyle6/scanners/scanner34.py b/uncompyle6/scanners/scanner34.py index b04352e1..dbcd2de3 100644 --- a/uncompyle6/scanners/scanner34.py +++ b/uncompyle6/scanners/scanner34.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015 by Rocky Bernstein +# Copyright (c) 2015-2016 by Rocky Bernstein """ Python 3.4 bytecode scanner/deparser @@ -327,7 +327,7 @@ class Scanner34(scan3.Scanner3): if __name__ == "__main__": co = inspect.currentframe().f_code - tokens, customize = Scanner34().disassemble(co) + tokens, customize = Scanner34(3.4).disassemble(co) for t in tokens: print(t) pass diff --git a/uncompyle6/scanners/scanner35.py b/uncompyle6/scanners/scanner35.py new file mode 100644 index 00000000..c05fceb3 --- /dev/null +++ b/uncompyle6/scanners/scanner35.py @@ -0,0 +1,333 @@ +# Copyright (c) 2016 by Rocky Bernstein +""" +Python 3.5 bytecode scanner/deparser + +This overlaps Python's 3.5's dis module, and in fact in some cases +we just fall back to that. But the intent is that it can be run from +Python 2 and other versions of Python. Also, we save token information +for later use in deparsing. +""" + +from __future__ import print_function + +import dis, inspect +from array import array +import uncompyle6.scanners.scanner3 as scan3 + +from uncompyle6 import PYTHON_VERSION +from uncompyle6.code import iscode +from uncompyle6.scanner import Token + +# Get all the opcodes into globals +globals().update(dis.opmap) + +import uncompyle6.opcodes.opcode_35 +# verify uses JUMP_OPs from here +JUMP_OPs = uncompyle6.opcodes.opcode_35.JUMP_OPs + +from uncompyle6.opcodes.opcode_35 import * + +class Scanner35(scan3.Scanner3): + + def disassemble(self, co, classname=None, code_objects={}): + fn = self.disassemble_built_in if PYTHON_VERSION == 3.4 \ + else self.disassemble_generic + return fn(co, classname, code_objects=code_objects) + + def disassemble_built_in(self, co, classname=None, + code_objects={}): + # Container for tokens + tokens = [] + customize = {} + self.code = array('B', co.co_code) + self.build_lines_data(co) + self.build_prev_op() + + # Get jump targets + # Format: {target offset: [jump offsets]} + jump_targets = self.find_jump_targets() + bytecode = dis.Bytecode(co) + + # self.lines contains (block,addrLastInstr) + if classname: + classname = '_' + classname.lstrip('_') + '__' + + def unmangle(name): + if name.startswith(classname) and name[-2:] != '__': + return name[len(classname) - 2:] + return name + + # free = [ unmangle(name) for name in (co.co_cellvars + co.co_freevars) ] + # names = [ unmangle(name) for name in co.co_names ] + # varnames = [ unmangle(name) for name in co.co_varnames ] + else: + # free = co.co_cellvars + co.co_freevars + # names = co.co_names + # varnames = co.co_varnames + pass + + # Scan for assertions. Later we will + # turn 'LOAD_GLOBAL' to 'LOAD_ASSERT' for those + # assertions + self.load_asserts = set() + bs = list(bytecode) + n = len(bs) + for i in range(n): + inst = bs[i] + if inst.opname == 'POP_JUMP_IF_TRUE' and i+1 < n: + next_inst = bs[i+1] + if (next_inst.opname == 'LOAD_GLOBAL' and + next_inst.argval == 'AssertionError'): + self.load_asserts.add(next_inst.offset) + + for inst in bytecode: + if inst.offset in jump_targets: + jump_idx = 0 + for jump_offset in jump_targets[inst.offset]: + tokens.append(Token('COME_FROM', None, repr(jump_offset), + offset='%s_%s' % (inst.offset, jump_idx))) + jump_idx += 1 + pass + pass + + pattr = inst.argrepr + opname = inst.opname + + if opname in ['LOAD_CONST']: + const = inst.argval + if iscode(const): + if const.co_name == '': + opname = 'LOAD_LAMBDA' + elif const.co_name == '': + opname = 'LOAD_GENEXPR' + elif const.co_name == '': + opname = 'LOAD_DICTCOMP' + elif const.co_name == '': + opname = 'LOAD_SETCOMP' + elif const.co_name == '': + opname = 'LOAD_LISTCOMP' + # verify() uses 'pattr' for comparison, since 'attr' + # now holds Code(const) and thus can not be used + # for comparison (todo: think about changing this) + # pattr = 'code_object @ 0x%x %s->%s' %\ + # (id(const), const.co_filename, const.co_name) + pattr = '' + else: + pattr = const + pass + elif opname in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET', 'BUILD_SLICE', + 'UNPACK_SEQUENCE', + 'MAKE_FUNCTION', 'MAKE_CLOSURE', + 'DUP_TOPX', 'RAISE_VARARGS' + ): + # if opname == 'BUILD_TUPLE' and \ + # self.code[self.prev[offset]] == LOAD_CLOSURE: + # continue + # else: + # op_name = '%s_%d' % (op_name, oparg) + # if opname != BUILD_SLICE: + # customize[op_name] = oparg + opname = '%s_%d' % (opname, inst.argval) + if inst.opname != 'BUILD_SLICE': + customize[opname] = inst.argval + + elif opname == 'JUMP_ABSOLUTE': + pattr = inst.argval + target = self.get_target(inst.offset) + if target < inst.offset: + if (inst.offset in self.stmts and + self.code[inst.offset+3] not in (END_FINALLY, POP_BLOCK) + and offset not in self.not_continue): + opname = 'CONTINUE' + else: + opname = 'JUMP_BACK' + + elif inst.offset in self.load_asserts: + opname = 'LOAD_ASSERT' + + tokens.append( + Token( + type_ = opname, + attr = inst.argval, + pattr = pattr, + offset = inst.offset, + linestart = inst.starts_line, + ) + ) + pass + return tokens, {} + + # FIXME: merge with scanner3 code + def detect_structure(self, offset): + """ + Detect structures and their boundaries to fix optimizied jumps + in python2.3+ + """ + code = self.code + op = code[offset] + # Detect parent structure + parent = self.structs[0] + start = parent['start'] + end = parent['end'] + + # Pick inner-most parent for our offset + for struct in self.structs: + curent_start = struct['start'] + curent_end = struct['end'] + if (curent_start <= offset < curent_end) and (curent_start >= start and curent_end <= end): + start = curent_start + end = curent_end + parent = struct + pass + + if op in (POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE): + start = offset + self.op_size(op) + target = self.get_target(offset) + rtarget = self.restrict_to_parent(target, parent) + prev_op = self.prev_op + + # Do not let jump to go out of parent struct bounds + if target != rtarget and parent['type'] == 'and/or': + self.fixed_jumps[offset] = rtarget + return + + # Does this jump to right after another cond jump? + # If so, it's part of a larger conditional + if (code[prev_op[target]] in (JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP, + POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE)) and (target > offset): + self.fixed_jumps[offset] = prev_op[target] + self.structs.append({'type': 'and/or', + 'start': start, + 'end': prev_op[target]}) + return + # Is it an and inside if block + if op == POP_JUMP_IF_FALSE: + # Search for other POP_JUMP_IF_FALSE targetting the same op, + # in current statement, starting from current offset, and filter + # everything inside inner 'or' jumps and midline ifs + match = self.rem_or(start, self.next_stmt[offset], POP_JUMP_IF_FALSE, target) + match = self.remove_mid_line_ifs(match) + # If we still have any offsets in set, start working on it + if match: + if (code[prev_op[rtarget]] in (JUMP_FORWARD, JUMP_ABSOLUTE) and prev_op[rtarget] not in self.stmts and + self.restrict_to_parent(self.get_target(prev_op[rtarget]), parent) == rtarget): + if (code[prev_op[prev_op[rtarget]]] == JUMP_ABSOLUTE and self.remove_mid_line_ifs([offset]) and + target == self.get_target(prev_op[prev_op[rtarget]]) and + (prev_op[prev_op[rtarget]] not in self.stmts or self.get_target(prev_op[prev_op[rtarget]]) > prev_op[prev_op[rtarget]]) and + 1 == len(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], (POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE), target)))): + pass + elif (code[prev_op[prev_op[rtarget]]] == RETURN_VALUE and self.remove_mid_line_ifs([offset]) and + 1 == (len(set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], + (POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE), target))) | + set(self.remove_mid_line_ifs(self.rem_or(start, prev_op[prev_op[rtarget]], + (POP_JUMP_IF_FALSE, POP_JUMP_IF_TRUE, JUMP_ABSOLUTE), + prev_op[rtarget], True)))))): + pass + else: + fix = None + jump_ifs = self.all_instr(start, self.next_stmt[offset], POP_JUMP_IF_FALSE) + last_jump_good = True + for j in jump_ifs: + if target == self.get_target(j): + if self.lines[j].next == j + 3 and last_jump_good: + fix = j + break + else: + last_jump_good = False + self.fixed_jumps[offset] = fix or match[-1] + return + else: + self.fixed_jumps[offset] = match[-1] + return + # op == POP_JUMP_IF_TRUE + else: + next = self.next_stmt[offset] + if prev_op[next] == offset: + pass + elif code[next] in (JUMP_FORWARD, JUMP_ABSOLUTE) and target == self.get_target(next): + if code[prev_op[next]] == POP_JUMP_IF_FALSE: + if code[next] == JUMP_FORWARD or target != rtarget or code[prev_op[prev_op[rtarget]]] not in (JUMP_ABSOLUTE, RETURN_VALUE): + self.fixed_jumps[offset] = prev_op[next] + return + elif (code[next] == JUMP_ABSOLUTE and code[target] in (JUMP_ABSOLUTE, JUMP_FORWARD) and + self.get_target(target) == self.get_target(next)): + self.fixed_jumps[offset] = prev_op[next] + return + + # Don't add a struct for a while test, it's already taken care of + if offset in self.ignore_if: + return + + if (code[prev_op[rtarget]] == JUMP_ABSOLUTE and prev_op[rtarget] in self.stmts and + prev_op[rtarget] != offset and prev_op[prev_op[rtarget]] != offset and + not (code[rtarget] == JUMP_ABSOLUTE and code[rtarget+3] == POP_BLOCK and code[prev_op[prev_op[rtarget]]] != JUMP_ABSOLUTE)): + rtarget = prev_op[rtarget] + + # Does the if jump just beyond a jump op, then this is probably an if statement + if code[prev_op[rtarget]] in (JUMP_ABSOLUTE, JUMP_FORWARD): + if_end = self.get_target(prev_op[rtarget]) + + # Is this a loop not an if? + if (if_end < prev_op[rtarget]) and (code[prev_op[if_end]] == SETUP_LOOP): + if(if_end > start): + return + + end = self.restrict_to_parent(if_end, parent) + + self.structs.append({'type': 'if-then', + 'start': start, + 'end': prev_op[rtarget]}) + self.not_continue.add(prev_op[rtarget]) + + if rtarget < end: + self.structs.append({'type': 'if-else', + 'start': rtarget, + 'end': end}) + elif code[prev_op[rtarget]] == RETURN_VALUE: + self.structs.append({'type': 'if-then', + 'start': start, + 'end': rtarget}) + self.return_end_ifs.add(prev_op[rtarget]) + + elif op in (JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP): + target = self.get_target(offset) + if target > offset: + unop_target = self.last_instr(offset, target, JUMP_FORWARD, target) + if unop_target and code[unop_target+3] != ROT_TWO: + self.fixed_jumps[offset] = unop_target + else: + self.fixed_jumps[offset] = self.restrict_to_parent(target, parent) + + def next_except_jump(self, start): + """ + Return the next jump that was generated by an except SomeException: + construct in a try...except...else clause or None if not found. + """ + + if self.code[start] == DUP_TOP: + except_match = self.first_instr(start, len(self.code), POP_JUMP_IF_FALSE) + if except_match: + jmp = self.prev_op[self.get_target(except_match)] + self.ignore_if.add(except_match) + self.not_continue.add(jmp) + return jmp + + count_END_FINALLY = 0 + count_SETUP_ = 0 + for i in self.op_range(start, len(self.code)): + op = self.code[i] + if op == END_FINALLY: + if count_END_FINALLY == count_SETUP_: + assert self.code[self.prev_op[i]] in (JUMP_ABSOLUTE, JUMP_FORWARD, RETURN_VALUE) + self.not_continue.add(self.prev_op[i]) + return self.prev_op[i] + count_END_FINALLY += 1 + elif op in (SETUP_EXCEPT, SETUP_WITH, SETUP_FINALLY): + count_SETUP_ += 1 + +if __name__ == "__main__": + co = inspect.currentframe().f_code + tokens, customize = Scanner35(3.5).disassemble(co) + for t in tokens: + print(t) + pass