diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 35f7740b..3064cf35 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -274,12 +274,12 @@ class Python2Parser(PythonParser): 'LOAD', 'LOOKUP', 'MAKE', 'SETUP', 'RAISE', 'UNPACK')) - # Opcode names in the custom_ops_seen set have rules that get added + # Opcode names in the custom_seen_ops set have rules that get added # unconditionally and the rules are constant. So they need to be done # only once and if we see the opcode a second we don't have to consider # adding more rules. # - custom_ops_seen = set() + custom_seen_ops = set() for i, token in enumerate(tokens): opname = token.kind @@ -287,7 +287,7 @@ class Python2Parser(PythonParser): # Do a quick breakout before testing potentially # each of the dozen or so instruction in if elif. if (opname[:opname.find('_')] not in customize_instruction_basenames - or opname in custom_ops_seen): + or opname in custom_seen_ops): continue opname_base = opname[:opname.rfind('_')] @@ -355,32 +355,32 @@ class Python2Parser(PythonParser): + 'expr ' * nak + opname elif opname == 'CONTINUE_LOOP': self.addRule('continue ::= CONTINUE_LOOP', nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'DELETE_ATTR': self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'DELETE_DEREF': self.addRule(""" stmt ::= del_deref_stmt del_deref_stmt ::= DELETE_DEREF """, nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'DELETE_SUBSCR': self.addRule(""" del_stmt ::= delete_subscr delete_subscr ::= expr expr DELETE_SUBSCR """, nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'GET_ITER': self.addRule(""" expr ::= get_iter attribute ::= expr GET_ITER """, nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'): # FIXME: remove these conditions if they are not needed. @@ -413,18 +413,18 @@ class Python2Parser(PythonParser): expr ::= attribute attribute ::= expr LOAD_ATTR """, nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'LOAD_LISTCOMP': self.addRule("expr ::= listcomp", nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'LOAD_SETCOMP': self.add_unique_rules([ "expr ::= set_comp", "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1" ], customize) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'LOOKUP_METHOD': # A PyPy speciality - DRY with parse3 @@ -433,7 +433,7 @@ class Python2Parser(PythonParser): attribute ::= expr LOOKUP_METHOD """, nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname_base == 'MAKE_FUNCTION': if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': @@ -482,7 +482,7 @@ class Python2Parser(PythonParser): "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy", "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM" ], customize) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname == 'SETUP_FINALLY': if 'PyPy' in customize: @@ -491,13 +491,13 @@ class Python2Parser(PythonParser): tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY suite_stmts_opt END_FINALLY""", nop_func) - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) continue elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) rule = 'unpack ::= ' + opname + ' store' * token.attr elif opname_base == 'UNPACK_LIST': - custom_ops_seen.add(opname) + custom_seen_ops.add(opname) rule = 'unpack_list ::= ' + opname + ' store' * token.attr else: continue diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 2e54cc16..794f8f2e 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -476,9 +476,7 @@ class Python3Parser(PythonParser): self.addRule(rule, nop_func) return - def custom_classfunc_rule(self, opname, token, customize, - possible_class_decorator, - seen_GET_AWAITABLE_YIELD_FROM, next_token): + def custom_classfunc_rule(self, opname, token, customize, next_token): """ call ::= expr {expr}^n CALL_FUNCTION_n call ::= expr {expr}^n CALL_FUNCTION_VAR_n @@ -508,7 +506,7 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, token.kind, uniq_param, customize) - if possible_class_decorator: + if 'LOAD_BUILD_CLASS' in self.seen_ops: if (next_token == 'CALL_FUNCTION' and next_token.attr == 1 and args_pos > 1): rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d' @@ -556,9 +554,9 @@ class Python3Parser(PythonParser): customize_instruction_basenames = frozenset( ('BUILD', 'CALL', 'CONTINUE', 'DELETE', 'GET', 'JUMP', 'LOAD', 'LOOKUP', 'MAKE', - 'RAISE', 'UNPACK')) + 'RETURN', 'RAISE', 'UNPACK')) - # Opcode names in the custom_ops_seen set have rules that get added + # Opcode names in the custom_ops_processed set have rules that get added # unconditionally and the rules are constant. So they need to be done # only once and if we see the opcode a second we don't have to consider # adding more rules. @@ -566,18 +564,13 @@ class Python3Parser(PythonParser): # Note: BUILD_TUPLE_UNPACK_WITH_CALL gets considered by # default because it starts with BUILD. So we'll set to ignore it from # the start. - custom_ops_seen = set(('BUILD_TUPLE_UNPACK_WITH_CALL',)) - - # In constrast to custom_ops_seen, seen_xxx rules here are part of some - # other rule; so if we see them a second time we still have to loop - # over customization - seen_LOAD_BUILD_CLASS = False - seen_GET_AWAITABLE_YIELD_FROM = False - - # This is used in parse36.py as well as here - self.seen_LOAD_DICTCOMP = False - self.seen_LOAD_SETCOMP = False + custom_ops_processed = set(('BUILD_TUPLE_UNPACK_WITH_CALL',)) + # A set of instruction operation names that exist in the token stream. + # We use this customize the grammar that we create. + self.seen_ops = {t.kind for t in tokens} + self.seen_op_basenames = {opname[:opname.rfind('_')] for opname in self.seen_ops} + from trepan.api import debug; debug() # Loop over instructions adding custom grammar rules based on # a specific instruction seen. @@ -595,8 +588,10 @@ class Python3Parser(PythonParser): return_lambda LAMBDA_MARKER """, nop_func) - has_get_iter_call_function1 = False n = len(tokens) + + # Determine if we have an iteration CALL_FUNCTION_1. + has_get_iter_call_function1 = False max_branches = 0 for i, token in enumerate(tokens): if token == 'GET_ITER' and i < n-2 and self.call_fn_name(tokens[i+1]) == 'CALL_FUNCTION_1': @@ -605,7 +600,6 @@ class Python3Parser(PythonParser): elif (token == 'GET_AWAITABLE' and i < n-3 and tokens[i+1] == 'LOAD_CONST' and tokens[i+2] == 'YIELD_FROM'): max_branches += 1 - seen_GET_AWAITABLE_YIELD_FROM = True if max_branches > 2: break @@ -615,7 +609,7 @@ class Python3Parser(PythonParser): # Do a quick breakout before testing potentially # each of the dozen or so instruction in if elif. if (opname[:opname.find('_')] not in customize_instruction_basenames - or opname in custom_ops_seen): + or opname in custom_ops_processed): continue opname_base = opname[:opname.rfind('_')] @@ -658,7 +652,7 @@ class Python3Parser(PythonParser): # FIXME: really we need a combination of dict_entry-like things. # It just so happens the most common case is not to mix # dictionary comphensions with dictionary, elements - if self.seen_LOAD_DICTCOMP: + if 'LOAD_DICTCOMP' in self.seen_ops: rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname) self.addRule(rule, nop_func) rule = """ @@ -737,9 +731,9 @@ class Python3Parser(PythonParser): """ self.addRule(rule, nop_func) - self.custom_classfunc_rule(opname, token, customize, - seen_LOAD_BUILD_CLASS, - seen_GET_AWAITABLE_YIELD_FROM, tokens[i+1]) + self.custom_classfunc_rule(opname, token, customize, tokens[i+1]) + # Note: don't add to custom_ops_processed. + elif opname_base == 'CALL_METHOD': # PyPy only - DRY with parse2 @@ -754,31 +748,31 @@ class Python3Parser(PythonParser): self.add_unique_rule(rule, opname, token.attr, customize) elif opname == 'CONTINUE': self.addRule('continue ::= CONTINUE', nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'CONTINUE_LOOP': self.addRule('continue ::= CONTINUE_LOOP', nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'DELETE_ATTR': self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'DELETE_DEREF': self.addRule(""" stmt ::= del_deref_stmt del_deref_stmt ::= DELETE_DEREF """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'DELETE_SUBSCR': self.addRule(""" del_stmt ::= delete_subscr delete_subscr ::= expr expr DELETE_SUBSCR """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'GET_ITER': self.addRule(""" expr ::= get_iter attribute ::= expr GET_ITER """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'JUMP_IF_NOT_DEBUG': v = token.attr self.addRule(""" @@ -793,40 +787,42 @@ class Python3Parser(PythonParser): LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM, """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'LOAD_BUILD_CLASS': - seen_LOAD_BUILD_CLASS = True self.custom_build_class_rule(opname, i, token, tokens, customize) + # Note: don't add to custom_ops_processed. elif opname == 'LOAD_CLASSDEREF': # Python 3.4+ self.addRule("expr ::= LOAD_CLASSDEREF", nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'LOAD_CLASSNAME': self.addRule("expr ::= LOAD_CLASSNAME", nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'LOAD_DICTCOMP': - self.seen_LOAD_DICTCOMP = True if has_get_iter_call_function1: rule_pat = ("dict_comp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr " "GET_ITER CALL_FUNCTION_1") self.add_make_function_rule(rule_pat, opname, token.attr, customize) - # listcomp is a custom Python3 rule + pass + custom_ops_processed.add(opname) elif opname == 'LOAD_ATTR': self.addRule(""" expr ::= attribute attribute ::= expr LOAD_ATTR """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'LOAD_LISTCOMP': self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize) + custom_ops_processed.add(opname) elif opname == 'LOAD_SETCOMP': - self.seen_LOAD_SETCOMP = True # Should this be generalized and put under MAKE_FUNCTION? if has_get_iter_call_function1: self.addRule("expr ::= set_comp", nop_func) rule_pat = ("set_comp ::= LOAD_SETCOMP %sMAKE_FUNCTION_0 expr " "GET_ITER CALL_FUNCTION_1") self.add_make_function_rule(rule_pat, opname, token.attr, customize) + pass + custom_ops_processed.add(opname) elif opname == 'LOOKUP_METHOD': # A PyPy speciality - DRY with parse3 self.addRule(""" @@ -834,12 +830,12 @@ class Python3Parser(PythonParser): attribute ::= expr LOOKUP_METHOD """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname.startswith('MAKE_CLOSURE'): # DRY with MAKE_FUNCTION # Note: this probably doesn't handle kwargs proprerly - if opname == 'MAKE_CLOSURE_0' and self.seen_LOAD_DICTCOMP: + if opname == 'MAKE_CLOSURE_0' and 'LOAD_DICTCOMP' in self.seen_ops: # Is there something general going on here? # Note that 3.6+ doesn't do this, but we'll remove # this rule in parse36.py @@ -1073,25 +1069,25 @@ class Python3Parser(PythonParser): self.addRule(""" return_lambda ::= ret_expr RETURN_VALUE_LAMBDA """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'RAISE_VARARGS_0': self.addRule(""" stmt ::= raise_stmt0 raise_stmt0 ::= RAISE_VARARGS_0 """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'RAISE_VARARGS_1': self.addRule(""" stmt ::= raise_stmt1 raise_stmt1 ::= expr RAISE_VARARGS_1 """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname == 'RAISE_VARARGS_2': self.addRule(""" stmt ::= raise_stmt2 raise_stmt2 ::= expr expr RAISE_VARARGS_2 """, nop_func) - custom_ops_seen.add(opname) + custom_ops_processed.add(opname) elif opname_base in ('UNPACK_EX',): before_count, after_count = token.attr rule = 'unpack ::= ' + opname + ' store' * (before_count + after_count + 1) @@ -1102,6 +1098,10 @@ class Python3Parser(PythonParser): elif opname_base == 'UNPACK_LIST': rule = 'unpack_list ::= ' + opname + ' store' * token.attr self.addRule(rule, nop_func) + custom_ops_processed.add(opname) + pass + pass + self.check_reduce['aug_assign1'] = 'AST' self.check_reduce['aug_assign2'] = 'AST' self.check_reduce['while1stmt'] = 'noAST' diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 382fb489..3d46230f 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -200,10 +200,7 @@ class Python35Parser(Python34Parser): pass return - def custom_classfunc_rule(self, opname, token, customize, - seen_LOAD_BUILD_CLASS, - seen_GET_AWAITABLE_YIELD_FROM, - *args): + def custom_classfunc_rule(self, opname, token, customize, *args): args_pos, args_kw = self.get_pos_kw(token) # Additional exprs for * and ** args: @@ -214,7 +211,7 @@ class Python35Parser(Python34Parser): nak = ( len(opname)-len('CALL_FUNCTION') ) // 3 uniq_param = args_kw + args_pos - if seen_GET_AWAITABLE_YIELD_FROM: + if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops): rule = ('async_call ::= expr ' + ('pos_arg ' * args_pos) + ('kwarg ' * args_kw) + @@ -243,10 +240,7 @@ class Python35Parser(Python34Parser): # zero or not in creating a template rule. self.add_unique_rule(rule, token.kind, args_pos, customize) else: - super(Python35Parser, self).custom_classfunc_rule(opname, token, customize, - seen_LOAD_BUILD_CLASS, - seen_GET_AWAITABLE_YIELD_FROM, - *args) + super(Python35Parser, self).custom_classfunc_rule(opname, token, customize, *args) class Python35ParserSingle(Python35Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 86934269..5788863e 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -108,7 +108,7 @@ class Python36Parser(Python35Parser): """ self.add_unique_doc_rules(rules_str, customize) elif opname == 'MAKE_FUNCTION_8': - if self.seen_LOAD_DICTCOMP: + if 'LOAD_DICTCOMP' in self.seen_ops: # Is there something general going on here? rule = """ dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST @@ -116,7 +116,7 @@ class Python36Parser(Python35Parser): GET_ITER CALL_FUNCTION_1 """ self.addRule(rule, nop_func) - elif self.seen_LOAD_SETCOMP: + elif 'LOAD_SETCOMP' in self.seen_ops: rule = """ set_comp ::= load_closure LOAD_SETCOMP LOAD_CONST MAKE_FUNCTION_8 expr @@ -192,9 +192,7 @@ class Python36Parser(Python35Parser): """ self.addRule(rules_str, nop_func) - def custom_classfunc_rule(self, opname, token, customize, - possible_class_decorator, - seen_GET_AWAITABLE_YIELD_FROM, next_token): + def custom_classfunc_rule(self, opname, token, customize, next_token): args_pos, args_kw = self.get_pos_kw(token) @@ -206,7 +204,7 @@ class Python36Parser(Python35Parser): nak = ( len(opname)-len('CALL_FUNCTION') ) // 3 uniq_param = args_kw + args_pos - if seen_GET_AWAITABLE_YIELD_FROM: + if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops): rule = ('async_call ::= expr ' + ('pos_arg ' * args_pos) + ('kwarg ' * args_kw) + @@ -261,11 +259,7 @@ class Python36Parser(Python35Parser): """, nop_func) pass else: - super(Python36Parser, self).custom_classfunc_rule(opname, token, - customize, - possible_class_decorator, - seen_GET_AWAITABLE_YIELD_FROM, - next_token) + super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token) def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python36Parser, diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index b1334b11..50425844 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -156,6 +156,7 @@ class Scanner3(Scanner): self.varargs_ops = frozenset(varargs_ops) # FIXME: remove the above in favor of: # self.varargs_ops = frozenset(self.opc.hasvargs) + return def ingest(self, co, classname=None, code_objects={}, show_asm=None): """