diff --git a/test/bytecode_3.6/01_boolean.pyc b/test/bytecode_3.6/01_boolean.pyc new file mode 100644 index 00000000..7a17b22e Binary files /dev/null and b/test/bytecode_3.6/01_boolean.pyc differ diff --git a/test/bytecode_3.6/01_class.pyc b/test/bytecode_3.6/01_class.pyc new file mode 100644 index 00000000..a7785276 Binary files /dev/null and b/test/bytecode_3.6/01_class.pyc differ diff --git a/test/bytecode_3.6/01_map_unpack.pyc b/test/bytecode_3.6/01_map_unpack.pyc new file mode 100644 index 00000000..7faba437 Binary files /dev/null and b/test/bytecode_3.6/01_map_unpack.pyc differ diff --git a/test/bytecode_3.6/02_build_map_unpack_with_call.pyc b/test/bytecode_3.6/02_build_map_unpack_with_call.pyc index f6d778d9..a6de6140 100644 Binary files a/test/bytecode_3.6/02_build_map_unpack_with_call.pyc and b/test/bytecode_3.6/02_build_map_unpack_with_call.pyc differ diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 64ce55ad..3cb2e1c3 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -98,6 +98,7 @@ case $PYVERSION in 2.7) SKIP_TESTS=( [test_dis.py]=1 # We change line numbers - duh! + [test_doctest.py]=1 [test_grammar.py]=1 # Too many stmts. Handle large stmts [test_io.py]=1 # Test takes too long to run [test_ioctl.py]=1 # Test takes too long to run @@ -110,6 +111,7 @@ case $PYVERSION in [test_strtod.py]=1 [test_traceback.py]=1 [test_unicode.py]=1 + [test_zipfile64.py]=1 # Runs ok but takes 204 seconds # Syntax errors: # .pyenv/versions/2.7.14/lib/python2.7/mimify.pyc # .pyenv/versions/2.7.14/lib/python2.7/netrc.pyc diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index f6d89f1b..2b895e94 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -156,6 +156,15 @@ class PythonParser(GenericASTBuilder): else: raise ParserError(None, -1) + def get_pos_kw(self, token): + """Return then the number of positional parameters and + represented by the attr field of token""" + # Low byte indicates number of positional paramters, + # high byte number of keyword parameters + args_pos = token.attr & 0xff + args_kw = (token.attr >> 8) & 0xff + return args_pos, args_kw + def nonterminal(self, nt, args): n = len(args) @@ -209,7 +218,7 @@ class PythonParser(GenericASTBuilder): def p_stmt(self, args): """ - passstmt ::= + pass ::= _stmts ::= stmt+ @@ -224,7 +233,7 @@ class PythonParser(GenericASTBuilder): lastc_stmt ::= ifelsestmtc c_stmts_opt ::= c_stmts - c_stmts_opt ::= passstmt + c_stmts_opt ::= pass l_stmts ::= _stmts l_stmts ::= return_stmts @@ -238,7 +247,7 @@ class PythonParser(GenericASTBuilder): lastl_stmt ::= tryelsestmtl l_stmts_opt ::= l_stmts - l_stmts_opt ::= passstmt + l_stmts_opt ::= pass suite_stmts ::= _stmts suite_stmts ::= return_stmts @@ -247,7 +256,7 @@ class PythonParser(GenericASTBuilder): suite_stmts_opt ::= suite_stmts # passtmt is needed for semantic actions to add "pass" - suite_stmts_opt ::= passstmt + suite_stmts_opt ::= pass else_suite ::= suite_stmts else_suitel ::= l_stmts diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 72396fea..a0c59c26 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -248,27 +248,30 @@ class Python2Parser(PythonParser): JUMP_BACK """, nop_func) - # Refactor the FIXME below and use the list below - # # For a rough break out on the first word. This may - # # include instructions that don't need customization, - # # but we'll do a finer check after the rough breakout. - # customize_instruction_basenames = frozenset( - # ('BUILD', 'CALL', 'CONTINUE_LOOP', 'DELETE', - # 'EXEC_STMT', 'JUMP', 'LOAD', 'LOOKUP', - # 'MAKE', 'SETUP_EXCEPT', 'SETUP_FINALLY', - # 'UNPACK')) + # For a rough break out on the first word. This may + # include instructions that don't need customization, + # but we'll do a finer check after the rough breakout. + customize_instruction_basenames = frozenset( + ('BUILD', 'CALL', 'CONTINUE', + 'DELETE', 'DUP', 'EXEC', 'JUMP', + 'LOAD', 'LOOKUP', 'MAKE', 'SETUP', + 'RAISE', 'UNPACK')) for i, token in enumerate(tokens): opname = token.kind - # FIXME: remove the "v" thing in the code below - if opname in customize: - v = customize[opname] - else: + + # FIXME + if (opname[:opname.find('_')] + not in customize_instruction_basenames): + + # if opname not in customize: continue + opname_base = opname[:opname.rfind('_')] # The order of opname listed is roughly sorted below if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'): + v = token.attr thousands = (v//1024) thirty32s = ((v//32) % 32) if thirty32s > 0: @@ -301,9 +304,9 @@ class Python2Parser(PythonParser): 'dictcomp_func', 0, customize) else: - kvlist_n = "kvlist_%s" % v + kvlist_n = "kvlist_%s" % token.attr self.add_unique_rules([ - (kvlist_n + " ::=" + ' kv3' * v), + (kvlist_n + " ::=" + ' kv3' * token.attr), "dict ::= %s %s" % (opname, kvlist_n) ], customize) continue @@ -324,23 +327,24 @@ class Python2Parser(PythonParser): continue elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): - args_pos = (v & 0xff) # positional parameters - args_kw = (v >> 8) & 0xff # keyword parameters + + args_pos, args_kw = self.get_pos_kw(token) + # number of apply equiv arguments: nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3 rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ + 'expr ' * nak + opname elif opname_base == 'CALL_METHOD': # PyPy only - DRY with parse3 - args_pos = (v & 0xff) # positional parameters - args_kw = (v >> 8) & 0xff # keyword parameters + + args_pos, args_kw = self.get_pos_kw(token) + # number of apply equiv arguments: nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ + 'expr ' * nak + opname elif opname == 'CONTINUE_LOOP': - self.add_unique_rule('continue ::= CONTINUE_LOOP', - opname, v, customize) + self.addRule('continue ::= CONTINUE_LOOP', nop_func) continue elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'): # FIXME: remove these conditions if they are not needed. @@ -355,16 +359,18 @@ class Python2Parser(PythonParser): """, nop_func) continue elif opname == 'JUMP_IF_NOT_DEBUG': - self.add_unique_rules([ - 'jmp_true_false ::= POP_JUMP_IF_TRUE', - 'jmp_true_false ::= POP_JUMP_IF_FALSE', - "stmt ::= assert_pypy", - "stmt ::= assert2_pypy", - "assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false " - "LOAD_ASSERT RAISE_VARARGS_1 COME_FROM", - "assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false " - "LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM", - ], customize) + v = token.attr + self.addRule(""" + jmp_true_false ::= POP_JUMP_IF_TRUE + jmp_true_false ::= POP_JUMP_IF_FALSE + stmt ::= assert_pypy + stmt ::= assert2_pypy + assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false + LOAD_ASSERT RAISE_VARARGS_1 COME_FROM + assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM + """, nop_func) continue elif opname == 'LOAD_LISTCOMP': self.add_unique_rules([ @@ -380,29 +386,29 @@ class Python2Parser(PythonParser): elif opname == 'LOOKUP_METHOD': # A PyPy speciality - DRY with parse3 self.add_unique_rule("attribute ::= expr LOOKUP_METHOD", - opname, v, customize) + opname, token.attr, customize) continue elif opname_base == 'MAKE_FUNCTION': if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': self.addRule('mklambda ::= %s LOAD_LAMBDA %s' % - ('pos_arg '*v, opname), nop_func) - rule = 'mkfunc ::= %s LOAD_CONST %s' % ('expr '*v, opname) + ('pos_arg ' * token.attr, opname), nop_func) + rule = 'mkfunc ::= %s LOAD_CONST %s' % ('expr ' * token.attr, opname) elif opname_base == 'MAKE_CLOSURE': # FIXME: use add_unique_rules to tidy this up. if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' % - ('expr '*v, opname), nop_func) + ('expr ' * token.attr, opname), nop_func) if i > 0: prev_tok = tokens[i-1] if prev_tok == 'LOAD_GENEXPR': self.add_unique_rules([ ('generator_exp ::= %s load_closure LOAD_GENEXPR %s expr' ' GET_ITER CALL_FUNCTION_1' % - ('expr '*v, opname))], customize) + ('expr ' * token.attr, opname))], customize) pass self.add_unique_rules([ ('mkfunc ::= %s load_closure LOAD_CONST %s' % - ('expr '*v, opname))], customize) + ('expr '* token.attr, opname))], customize) if self.version >= 2.7: if i > 0: @@ -411,13 +417,13 @@ class Python2Parser(PythonParser): self.add_unique_rules([ ('dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr' ' GET_ITER CALL_FUNCTION_1' % - ('expr '*v, opname))], customize) + ('expr ' * token.attr, opname))], customize) elif prev_tok == 'LOAD_SETCOMP': self.add_unique_rules([ "expr ::= set_comp", ('set_comp ::= %s load_closure LOAD_SETCOMP %s expr' ' GET_ITER CALL_FUNCTION_1' % - ('expr '*v, opname)) + ('expr ' * token.attr, opname)) ], customize) pass pass @@ -443,13 +449,14 @@ class Python2Parser(PythonParser): ], customize) continue elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): - rule = 'unpack ::= ' + opname + ' store'*v + rule = 'unpack ::= ' + opname + ' store' * token.attr elif opname_base == 'UNPACK_LIST': - rule = 'unpack_list ::= ' + opname + ' store'*v + rule = 'unpack_list ::= ' + opname + ' store' * token.attr else: - raise Exception('unknown customize token %s' % opname) - self.add_unique_rule(rule, opname_base, v, customize) + continue + self.addRule(rule, nop_func) pass + self.check_reduce['aug_assign1'] = 'AST' self.check_reduce['aug_assign2'] = 'AST' self.check_reduce['_stmts'] = 'AST' diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 84aee87d..db4b6543 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -150,7 +150,7 @@ class Python3Parser(PythonParser): else_suite _come_froms # ifelsestmt ::= testexpr c_stmts_opt jump_forward_else - # passstmt _come_froms + # pass _come_froms ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec @@ -429,22 +429,38 @@ class Python3Parser(PythonParser): assert i < len(tokens), "build_class needs to find MAKE_FUNCTION or MAKE_CLOSURE" assert tokens[i+1].kind == 'LOAD_CONST', \ "build_class expecting CONST after MAKE_FUNCTION/MAKE_CLOSURE" + call_fn_tok = None for i in range(i, len(tokens)): - if tokens[i].kind == 'CALL_FUNCTION': + if tokens[i].kind.startswith('CALL_FUNCTION'): call_fn_tok = tokens[i] break - assert call_fn_tok, "build_class custom rule needs to find CALL_FUNCTION" + if not call_fn_tok: + raise RuntimeError("build_class custom rule for %s needs to find CALL_FUNCTION" + % opname) + # customize build_class rule # FIXME: What's the deal with the two rules? Different Python versions? # Different situations? Note that the above rule is based on the CALL_FUNCTION # token found, while this one doesn't. - call_function = self.call_fn_name(call_fn_tok) - args_pos, args_kw = self.get_pos_kw(call_fn_tok) - rule = ("build_class ::= LOAD_BUILD_CLASS mkfunc %s" - "%s" % (('expr ' * (args_pos - 1) + ('kwarg ' * args_kw)), - call_function)) - self.add_unique_rule(rule, opname, token.attr, customize) + if self.version < 3.6: + call_function = self.call_fn_name(call_fn_tok) + args_pos, args_kw = self.get_pos_kw(call_fn_tok) + rule = ("build_class ::= LOAD_BUILD_CLASS mkfunc %s" + "%s" % (('expr ' * (args_pos - 1) + ('kwarg ' * args_kw)), + call_function)) + else: + # 3.6+ handling + call_function = call_fn_tok.kind + if call_function.startswith("CALL_FUNCTION_KW"): + self.addRule("classdef ::= build_class_kw store", nop_func) + rule = ("build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" + % ('expr ' * (call_fn_tok.attr - 1), call_function)) + else: + call_function = self.call_fn_name(call_fn_tok) + rule = ("build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" + % ('expr ' * (call_fn_tok.attr - 1), call_function)) + self.addRule(rule, nop_func) return def custom_classfunc_rule(self, opname, token, customize, @@ -519,16 +535,6 @@ class Python3Parser(PythonParser): new_rule = rule % (('LOAD_CONST ') * 0) self.add_unique_rule(new_rule, opname, attr, customize) - def get_pos_kw(self, token): - """Return then the number of positional parameters and - represented by the attr field of token""" - # Low byte indicates number of positional paramters, - # high byte number of keyword parameters - args_pos = token.attr & 0xff - args_kw = (token.attr >> 8) & 0xff - return args_pos, args_kw - - def customize_grammar_rules(self, tokens, customize): """The base grammar we start out for a Python version even with the subclassing is, well, is pretty base. And we want it that way: lean and diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index 7cd21927..10a7c11b 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -76,7 +76,7 @@ class Python35Parser(Python34Parser): LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK JUMP_ABSOLUTE END_FINALLY JUMP_BACK - passstmt POP_BLOCK JUMP_ABSOLUTE + pass POP_BLOCK JUMP_ABSOLUTE COME_FROM_LOOP stmt ::= async_forelse_stmt diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 311b306e..dd59c685 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -102,11 +102,7 @@ class Scanner2(Scanner): # list of tokens/instructions tokens = [] - # "customize" is a dict whose keys are nonterminals - # and the value is the argument stack entries for that - # nonterminal. The count is a little hoaky. It is mostly - # not used, but sometimes it is. - # "customize" is a dict whose keys are nonterminals + # "customize" is in the process of going away here customize = {} if self.is_pypy: @@ -253,17 +249,9 @@ class Scanner2(Scanner): op_name = 'BUILD_MAP_n' else: op_name = '%s_%d' % (op_name, oparg) + pass + # FIXME: Figure out why this is needed and remove. customize[op_name] = oparg - elif self.is_pypy and op_name in frozenset( - """LOOKUP_METHOD JUMP_IF_NOT_DEBUG SETUP_EXCEPT SETUP_FINALLY""".split()): - # The value in the dict is in special cases in semantic actions, such - # as CALL_FUNCTION. The value is not used in these cases, so we put - # in arbitrary value 0. - customize[op_name] = 0 - elif op_name in """ - CONTINUE_LOOP EXEC_STMT LOAD_LISTCOMP LOAD_SETCOMP - """.split(): - customize[op_name] = 0 elif op == self.opc.JUMP_ABSOLUTE: # Further classify JUMP_ABSOLUTE into backward jumps # which are used in loops, and "CONTINUE" jumps which diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index 90ccdc98..b95d39d7 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -32,7 +32,7 @@ RETURN_NONE = AST('stmt', PASS = AST('stmts', [ AST('sstmt', [ AST('stmt', - [ AST('passstmt', [])])])]) + [ AST('pass', [])])])]) ASSIGN_DOC_STRING = lambda doc_string: \ AST('stmt', @@ -276,7 +276,7 @@ TABLE_DIRECT = { 'except_cond1': ( '%|except %c:\n', 1 ), 'except_suite': ( '%+%c%-%C', 0, (1, maxint, '') ), 'except_suite_finalize': ( '%+%c%-%C', 1, (3, maxint, '') ), - 'passstmt': ( '%|pass\n', ), + 'pass': ( '%|pass\n', ), 'STORE_FAST': ( '%{pattr}', ), 'kv': ( '%c: %c', 3, 1 ), 'kv2': ( '%c: %c', 1, 2 ), diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 720635ae..694a0e66 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -89,7 +89,7 @@ ExtractInfo = namedtuple("ExtractInfo", TABLE_DIRECT_FRAGMENT = { 'break': ( '%|%rbreak\n', ), 'continue ': ( '%|%rcontinue\n', ), - 'passstmt': ( '%|%rpass\n', ), + 'pass': ( '%|%rpass\n', ), 'raise_stmt0': ( '%|%rraise\n', ), 'import': ( '%|import %c%x\n', 2, (2, (0, 1)), ), 'importfrom': ( '%|from %[2]{pattr}%x import %c\n', (2, (0, 1)), 3), @@ -856,7 +856,7 @@ class FragmentsWalker(pysource.SourceWalker, object): def n__ifstmts_jump_exit(self, node): if len(node) > 1: if (node[0] == 'c_stmts_opt' and - node[0][0] == 'passstmt' and + node[0][0] == 'pass' and node[1].kind.startswith('JUMP_FORWARD')): self.set_pos_info(node[1], node[0][0].start, node[0][0].finish) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 1b4d34ac..947ec74a 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1787,6 +1787,11 @@ class SourceWalker(GenericASTTraversal, object): def print_super_classes3(self, node): n = len(node)-1 if node.kind != 'expr': + kwargs = None + # 3.6+ starts having this + if node[n].kind.startswith('CALL_FUNCTION_KW'): + kwargs = node[n-1].attr + assert isinstance(kwargs, tuple) assert node[n].kind.startswith('CALL_FUNCTION') for i in range(n-2, 0, -1): if not node[i].kind in ['expr', 'LOAD_CLASSNAME']: @@ -1798,11 +1803,21 @@ class SourceWalker(GenericASTTraversal, object): line_separator = ', ' sep = '' self.write('(') + j = 0 i += 2 - while i < n: + if kwargs: + # Last arg is tuple of keyword values: omit + l = n - 1 + else: + l = n + while i < l: value = self.traverse(node[i]) i += 1 self.write(sep, value) + # 3.6+ may have this + if kwargs: + self.write("=%s" % kwargs[j]) + j += 1 sep = line_separator pass pass