diff --git a/test/bytecode_2.3/03_if1.pyc b/test/bytecode_2.3/03_if1.pyc new file mode 100644 index 00000000..de82a585 Binary files /dev/null and b/test/bytecode_2.3/03_if1.pyc differ diff --git a/test/bytecode_2.7/02_while1else.pyc b/test/bytecode_2.7/02_while1else.pyc new file mode 100644 index 00000000..ecf63787 Binary files /dev/null and b/test/bytecode_2.7/02_while1else.pyc differ diff --git a/test/bytecode_2.7/09_whiletrue_bug.pyc b/test/bytecode_2.7/09_whiletrue_bug.pyc index 10a6b1c4..cd16276a 100644 Binary files a/test/bytecode_2.7/09_whiletrue_bug.pyc and b/test/bytecode_2.7/09_whiletrue_bug.pyc differ diff --git a/test/bytecode_3.1/03_if_1_else.pyc b/test/bytecode_3.1/03_if_1_else.pyc new file mode 100644 index 00000000..5085777c Binary files /dev/null and b/test/bytecode_3.1/03_if_1_else.pyc differ diff --git a/test/bytecode_3.1/04_def_annotate.pyc b/test/bytecode_3.1/04_def_annotate.pyc deleted file mode 100644 index c8a5c062..00000000 Binary files a/test/bytecode_3.1/04_def_annotate.pyc and /dev/null differ diff --git a/test/bytecode_3.3/02_while1else.pyc b/test/bytecode_3.3/02_while1else.pyc new file mode 100644 index 00000000..6ebb0b3d Binary files /dev/null and b/test/bytecode_3.3/02_while1else.pyc differ diff --git a/test/bytecode_3.4/04_def_annotate.pyc b/test/bytecode_3.4/04_def_annotate.pyc index 28fea024..376e2575 100644 Binary files a/test/bytecode_3.4/04_def_annotate.pyc and b/test/bytecode_3.4/04_def_annotate.pyc differ diff --git a/test/bytecode_3.5/02_while1else.pyc b/test/bytecode_3.5/02_while1else.pyc new file mode 100644 index 00000000..47d71166 Binary files /dev/null and b/test/bytecode_3.5/02_while1else.pyc differ diff --git a/test/bytecode_3.5/04_def_annotate.pyc b/test/bytecode_3.5/04_def_annotate.pyc index a18364b9..784db926 100644 Binary files a/test/bytecode_3.5/04_def_annotate.pyc and b/test/bytecode_3.5/04_def_annotate.pyc differ diff --git a/test/bytecode_3.5/09_whiletrue_bug.pyc b/test/bytecode_3.5/09_whiletrue_bug.pyc index 4c843d60..992ad726 100644 Binary files a/test/bytecode_3.5/09_whiletrue_bug.pyc and b/test/bytecode_3.5/09_whiletrue_bug.pyc differ diff --git a/test/simple_source/bug27+/03_if_1_else.py b/test/simple_source/bug27+/03_if_1_else.py new file mode 100644 index 00000000..bfdcae3c --- /dev/null +++ b/test/simple_source/bug27+/03_if_1_else.py @@ -0,0 +1 @@ +1 if 1 else __file__ diff --git a/test/simple_source/bug31/04_def_annotate.py b/test/simple_source/bug31/04_def_annotate.py index a42e2a09..52942dd5 100644 --- a/test/simple_source/bug31/04_def_annotate.py +++ b/test/simple_source/bug31/04_def_annotate.py @@ -17,3 +17,14 @@ def div(a: dict(type=float, help='the dividend'), ) -> dict(type=float, help='the result of dividing a by b'): """Divide a by b""" return a / b + +class TestSignatureObject(unittest.TestCase): + def test_signature_on_wkwonly(self): + def test(*, a:float, b:str) -> int: + pass + +class SupportsInt(_Protocol): + + @abstractmethod + def __int__(self) -> int: + pass diff --git a/test/simple_source/stmts/02_while1else.py b/test/simple_source/stmts/02_while1else.py new file mode 100644 index 00000000..a01efafa --- /dev/null +++ b/test/simple_source/stmts/02_while1else.py @@ -0,0 +1,8 @@ +# From Python-3.5.2/Lib/multiprocessing/connection.py + +def PipeClient(address): + while 1: + z = 2 + else: + raise + return diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index acd80d8a..e4c6f623 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -512,8 +512,12 @@ class PythonParser(GenericASTBuilder): expr ::= conditional conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr + expr ::= conditionalnot - conditionalnot ::= expr jmp_true expr _jump expr COME_FROM + conditionalnot ::= expr jmp_true expr _jump expr COME_FROM + + expr ::= conditionalTrue + conditionalTrue ::= expr JUMP_FORWARD expr COME_FROM ret_expr ::= expr ret_expr ::= ret_and diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 280e78c2..8a80a1f4 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -42,7 +42,8 @@ class Python2Parser(PythonParser): while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK COME_FROM while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK COME_FROM - while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suite COME_FROM + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite COME_FROM exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT diff --git a/uncompyle6/parsers/parse23.py b/uncompyle6/parsers/parse23.py index 160c4de1..b2072e16 100644 --- a/uncompyle6/parsers/parse23.py +++ b/uncompyle6/parsers/parse23.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016 Rocky Bernstein +# Copyright (c) 2016-2017 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # Copyright (c) 1999 John Aycock @@ -14,6 +14,17 @@ class Python23Parser(Python24Parser): def p_misc23(self, args): ''' + # Python 2.4 only adds something like the below for if 1: + # However we will just treat it as a noop (which of course messes up + # simple verify of bytecode. + # See also below in reduce_is_invalid where we check that the JUMP_FORWARD + # target matches the COME_FROM target + stmt ::= if1_stmt + if1_stmt ::= JUMP_FORWARD JUMP_IF_FALSE THEN POP_TOP COME_FROM + stmts + JUMP_FORWARD COME_FROM POP_TOP COME_FROM + + # Used to keep semantic positions the same across later versions # of Python _while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM @@ -33,6 +44,23 @@ class Python23Parser(Python24Parser): lc_body ::= LOAD_FAST expr LIST_APPEND ''' + def add_custom_rules(self, tokens, customize): + super(Python23Parser, self).add_custom_rules(tokens, customize) + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + invalid = super(Python24Parser, + self).reduce_is_invalid(rule, ast, + tokens, first, last) + if invalid: + return invalid + + # FiXME: this code never gets called... + lhs = rule[0] + if lhs == 'nop_stmt': + return not int(tokens[first].pattr) == tokens[last].offset + + return False + class Python23ParserSingle(Python23Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse24.py b/uncompyle6/parsers/parse24.py index 92a798a2..ac8ff71d 100644 --- a/uncompyle6/parsers/parse24.py +++ b/uncompyle6/parsers/parse24.py @@ -14,6 +14,15 @@ class Python24Parser(Python25Parser): def p_misc24(self, args): ''' + # Python 2.4 only adds something like the below for if 1: + # However we will just treat it as a noop (which of course messes up + # simple verify of bytecode. + # See also below in reduce_is_invalid where we check that the JUMP_FORWARD + # target matches the COME_FROM target + stmt ::= nop_stmt + nop_stmt ::= JUMP_FORWARD POP_TOP COME_FROM + + # 2.5+ has two LOAD_CONSTs, one for the number '.'s in a relative import # keep positions similar to simplify semantic actions @@ -37,6 +46,25 @@ class Python24Parser(Python25Parser): gen_comp_body ::= expr YIELD_VALUE ''' + def add_custom_rules(self, tokens, customize): + super(Python24Parser, self).add_custom_rules(tokens, customize) + if self.version == 2.4: + self.check_reduce['nop_stmt'] = 'tokens' + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + invalid = super(Python24Parser, + self).reduce_is_invalid(rule, ast, + tokens, first, last) + if invalid: + return invalid + + # FiXME: this code never gets called... + lhs = rule[0] + if lhs == 'nop_stmt': + return not int(tokens[first].pattr) == tokens[last].offset + + return False + class Python24ParserSingle(Python24Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index bb556472..7f6b326c 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -154,7 +154,7 @@ class Python3Parser(PythonParser): # of missing "else" clauses. Therefore we include grammar # rules with and without ELSE. - ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM + ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec @@ -273,12 +273,14 @@ class Python3Parser(PythonParser): stmt ::= funcdef_annotate funcdef_annotate ::= mkfunc_annotate designator + mkfuncdeco0 ::= mkfunc_annotate + # This has the annotation value. # LOAD_NAME is used in an annotation type like # int, float, str annotate_arg ::= LOAD_NAME # LOAD_CONST is used in an annotation string - annotate_arg ::= LOAD_CONST + annotate_arg ::= expr # This stores the tuple of parameter names # that have been annotated @@ -379,6 +381,8 @@ class Python3Parser(PythonParser): while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK + else_suite COME_FROM_LOOP # FIXME: investigate - can code really produce a NOP? whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP @@ -403,7 +407,6 @@ class Python3Parser(PythonParser): conditional ::= expr jmp_false expr jf_else expr COME_FROM conditionalnot ::= expr jmp_true expr jf_else expr COME_FROM - expr ::= LOAD_CLASSNAME # Python 3.4+ @@ -708,9 +711,6 @@ class Python3Parser(PythonParser): ('pos_arg ' * args_pos, opname)) self.add_unique_rule(rule, opname, token.attr, customize) if opname.startswith('MAKE_FUNCTION_A'): - # rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' % - # ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname)) - self.add_unique_rule(rule, opname, token.attr, customize) if self.version >= 3.3: rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' % (('pos_arg ' * (args_pos)), diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index c5383f78..e60382c6 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -19,8 +19,11 @@ class Python35Parser(Python34Parser): # I'm sure by the time Python 4 comes around these will be turned # into special opcodes - while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK - POP_BLOCK COME_FROM_LOOP + while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK + POP_BLOCK COME_FROM_LOOP + while1stmt ::= SETUP_LOOP l_stmts POP_BLOCK COME_FROM_LOOP + while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK + POP_BLOCK else_suite COME_FROM_LOOP # Python 3.5+ Await statement stmt ::= await_stmt diff --git a/uncompyle6/scanners/scanner27.py b/uncompyle6/scanners/scanner27.py index 706d9a62..024253fc 100755 --- a/uncompyle6/scanners/scanner27.py +++ b/uncompyle6/scanners/scanner27.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, 2016 by Rocky Bernstein +# Copyright (c) 2015-2017 by Rocky Bernstein """ Python 2.7 bytecode ingester. @@ -46,7 +46,7 @@ class Scanner27(Scanner2): self.opc.DELETE_SLICE_2, self.opc.DELETE_SLICE_3, ]) - # opcodes with expect a variable number pushed values whose + # opcodes which expect a variable number pushed values and whose # count is in the opcode. For parsing we generally change the # opcode name to include that number. varargs_ops = set([ diff --git a/uncompyle6/semantics/check_ast.py b/uncompyle6/semantics/check_ast.py index 8bca0411..efea085d 100644 --- a/uncompyle6/semantics/check_ast.py +++ b/uncompyle6/semantics/check_ast.py @@ -10,7 +10,7 @@ before reduction and don't reduce when there is a problem. def checker(ast, in_loop, errors): in_loop = in_loop or ast.type in ('while1stmt', 'whileTruestmt', - 'whilestmt', 'whileelsestmt', + 'whilestmt', 'whileelsestmt', 'while1elsestmt', 'for_block') if ast.type in ('augassign1', 'augassign2') and ast[0][0] == 'and': text = str(ast) diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index ee233ae3..6570522b 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -170,6 +170,7 @@ TABLE_DIRECT = { 'or': ( '%c or %c', 0, 2 ), 'ret_or': ( '%c or %c', 0, 2 ), 'conditional': ( '%p if %p else %p', (2, 27), (0, 27), (4, 27)), + 'conditionalTrue': ( '%p if 1 else %p', (0, 27), (2, 27)), 'ret_cond': ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27)), 'conditionalnot': ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27)), 'ret_cond_not': ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27)), diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 443d64ad..f6e38c1f 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -77,7 +77,9 @@ from uncompyle6.semantics.consts import ( TABLE_DIRECT, escape, minint, MAP ) -from uncompyle6.semantics.make_function import find_all_globals, find_none +from uncompyle6.semantics.make_function import ( + find_all_globals, find_none, code_has_star_arg, code_has_star_star_arg +) from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG @@ -1668,10 +1670,10 @@ class FragmentsWalker(pysource.SourceWalker, object): name, default in map(lambda *tup:tup, *tup)] params.reverse() # back to correct order - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): params.append('*%s' % code.co_varnames[argc]) argc += 1 - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): params.append('**%s' % code.co_varnames[argc]) argc += 1 diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index a0fa0847..778f6dee 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, 2016 by Rocky Bernstein +# Copyright (c) 2015-2017 by Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel """ All the crazy things we have to do to handle Python functions @@ -40,6 +40,17 @@ def find_none(node): return True return False +# FIXME: put this in xdis +def code_has_star_arg(code): + """Return True iff + the code object has a variable positional parameter (*args-like)""" + return (code.co_flags & 4) != 0 + +def code_has_star_star_arg(code): + """Return True iff + The code object has a variable keyword parameter (**kwargs-like).""" + return (code.co_flags & 8) != 0 + # FIXME: DRY the below code... def make_function3_annotate(self, node, isLambda, nested=1, @@ -81,8 +92,7 @@ def make_function3_annotate(self, node, isLambda, nested=1, j = annotate_last-1 l = -len(node) while j >= l and node[j].type in ('annotate_arg' 'annotate_tuple'): - annotate_args[annotate_tup[i]] = (node[j][0].attr, - node[j][0] == 'LOAD_CONST') + annotate_args[annotate_tup[i]] = node[j][0] i -= 1 j -= 1 @@ -91,9 +101,12 @@ def make_function3_annotate(self, node, isLambda, nested=1, # positional args are before kwargs defparams = node[:args_node.attr[0]] pos_args, kw_args, annotate_argc = args_node.attr + if 'return' in annotate_args.keys(): + annotate_argc = len(annotate_args) - 1 else: defparams = node[:args_node.attr] kw_args = 0 + annotate_argc = 0 pass if 3.0 <= self.version <= 3.2: @@ -139,7 +152,7 @@ def make_function3_annotate(self, node, isLambda, nested=1, indent = ' ' * l line_number = self.line_number - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): self.write('*%s' % code.co_varnames[argc + kw_pairs]) argc += 1 @@ -184,10 +197,12 @@ def make_function3_annotate(self, node, isLambda, nested=1, else: suffix = ', ' + # self.println(indent, '#flags:\t', int(code.co_flags)) - if kw_args > 0: - if not (4 & code.co_flags): + if kw_args + annotate_argc > 0: + if not code_has_star_arg(code): if argc > 0: + self.write(", *, ") else: self.write("*, ") @@ -210,9 +225,29 @@ def make_function3_annotate(self, node, isLambda, nested=1, i += 1 pass pass + annotate_args = [] + for n in node: + if n == 'annotate_arg': + annotate_args.append(n[0]) + elif n == 'annotate_tuple': + t = n[0].attr + if t[-1] == 'return': + t = t[0:-1] + annotate_args = annotate_args[:-1] + pass + last = len(annotate_args) - 1 + for i in range(len(annotate_args)): + self.write("%s: " % (t[i])) + self.preorder(annotate_args[i]) + if i < last: + self.write(', ') + pass + pass + break + pass pass - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): if argc > 0: self.write(', ') self.write('**%s' % code.co_varnames[argc + kw_pairs]) @@ -295,6 +330,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): else: defparams = node[:args_node.attr] kw_args = 0 + annotate_argc = 0 pass lambda_index = None @@ -337,7 +373,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): name, default in map(lambda *tup:tup, *tup)] params.reverse() # back to correct order - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): params.append('*%s' % code.co_varnames[argc]) argc += 1 @@ -365,7 +401,7 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): break pass - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): if argc > 0: self.write(', ') self.write('**%s' % code.co_varnames[argc + kw_pairs]) @@ -478,7 +514,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): name, default in map(lambda *tup:tup, *tup)] params.reverse() # back to correct order - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): if self.version > 3.0: params.append('*%s' % code.co_varnames[argc + kw_pairs]) else: @@ -504,7 +540,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): indent = ' ' * l line_number = self.line_number - if 4 & code.co_flags: # flag 2 -> variable number of args + if code_has_star_arg(code): self.write('*%s' % code.co_varnames[argc + kw_pairs]) argc += 1 @@ -561,7 +597,7 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): pass pass - if 8 & code.co_flags: # flag 3 -> keyword args + if code_has_star_star_arg(code): if argc > 0: self.write(', ') self.write('**%s' % code.co_varnames[argc + kw_pairs]) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 94cf983f..2b12c2c2 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -216,6 +216,11 @@ class SourceWalker(GenericASTTraversal, object): 'importlist2': ( '%C', (0, maxint, ', ') ), }) if version <= 2.4: + if version == 2.3: + TABLE_DIRECT.update({ + 'if1_stmt': ( '%|if 1\n%+%c%-', 5 ) + }) + global NAME_MODULE NAME_MODULE = AST('stmt', [ AST('assign', @@ -1805,7 +1810,8 @@ class SourceWalker(GenericASTTraversal, object): str += '*%c, **%c)' # Python 3.5 only puts optional args (the VAR part) # lowest down the stack - if self.version == 3.5: + na = (v & 0xff) # positional parameters + if self.version == 3.5 and na == 0: if p2[2]: p2 = (2, -2, ', ') entry = (str, 0, p2, 1, -2) else: