From e39c6c7f0a6c8478888fb6cf4950b8876787fe73 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 18 Dec 2019 09:14:09 -0500 Subject: [PATCH] Fix "with as" indenting and decorated "async def" Misc other improvements: make check-short now is short --- test/Makefile | 2 +- test/bytecode_3.7/04_async.pyc | Bin 0 -> 425 bytes test/simple_source/bug37/04_async.py | 12 ++ test/stdlib/runtests.sh | 32 ++++- uncompyle6/semantics/consts.py | 8 ++ uncompyle6/semantics/customize3.py | 2 +- uncompyle6/semantics/customize35.py | 190 +++++++++++++++------------ 7 files changed, 161 insertions(+), 85 deletions(-) create mode 100644 test/bytecode_3.7/04_async.pyc create mode 100644 test/simple_source/bug37/04_async.py diff --git a/test/Makefile b/test/Makefile index 598154cb..b95180a3 100644 --- a/test/Makefile +++ b/test/Makefile @@ -23,7 +23,7 @@ COVER_DIR=../tmp/grammar-cover # Run short tests check-short: @$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \ - $(MAKE) check-bytecode-short + $(MAKE) check-bytecode-$${PYTHON_VERSION} # Run all tests check: diff --git a/test/bytecode_3.7/04_async.pyc b/test/bytecode_3.7/04_async.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b2e13b77e6b14589414332cd2d7e5a961698a46 GIT binary patch literal 425 zcmZ8e!Ab)$5S_`cwzgV&5U-y1VucojBBFv|F9r2rS&*>q##-DqB}uV`7Qy-zp4y|| z;a}wHsb_CJIa#ZS1Me}Dyf-r>r(0VEpr}`uH%|!g;fkN|C^;tRGLe7;5`2LvhM2G` z2)vl|S@^_7!aG5yL}QqN!5N!FgIwp7&lwWeR`qLrg2tP|Z(W+O1PN^7U6bQtrMrX{ zxyJaA*Dd2e_Ti`qrJtGw7;rgSL_fp!@ zNha&a-kDNT1-|vQNbhVemZTgSi$jZ}H&7eYcDbuN-F_;#mID=w%I%{dXos)4l7|9ecVRn-lrL`u=wraPM;lS1OAZv4x2yh%4Cp=BiHPc F0AEKaViN!W literal 0 HcmV?d00001 diff --git a/test/simple_source/bug37/04_async.py b/test/simple_source/bug37/04_async.py new file mode 100644 index 00000000..eb46010b --- /dev/null +++ b/test/simple_source/bug37/04_async.py @@ -0,0 +1,12 @@ +# from 3.7 test_contextlib_async.py +# Bugs were not adding "async" when a function is a decorator, +# and a misaligment when using "async with as". +@_async_test +async def test_enter(self): + self.assertIs(await manager.__aenter__(), manager) + + async with manager as context: + async with woohoo() as x: + x = 1 + y = 2 + assert manager is context diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 4b353583..617df0da 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -165,6 +165,7 @@ case $PYVERSION in SKIP_TESTS=( [test_atexit.py]=1 # [test_decorators.py]=1 # Control flow wrt "if elif" + [test_dis.py]=1 # We change line numbers - duh! ) if (( batch )) ; then # Fails in crontab environment? @@ -178,16 +179,43 @@ case $PYVERSION in SKIP_TESTS=( [test_contains.py]=1 # Code "while False: yield None" is optimized away in compilation [test_decorators.py]=1 # Control flow wrt "if elif" + [test_dis.py]=1 # We change line numbers - duh! [test_pow.py]=1 # Control flow wrt "continue" [test_quopri.py]=1 # Only fails on POWER ) ;; 3.7) SKIP_TESTS=( - [test_argparse.py]=1 # [test_ast.py]=1 # + [test_atexit.py]=1 # + [test_bdb.py]=1 # + [test_buffer.py]=1 # + [test_builtin.py]=1 # + [test_c_locale_coercion.py]=1 # Parse error + [test_cmdline.py]=1 # Interactive? + [test_codecs-3.7.py]=1 + [test_collections.py]=1 # Investigate syntax error: self.assertEqual(p, Point(**)) + [test_compare.py]=1 + [test_compile.py]=1 + [test_complex.py]=1 # Investigate: NameError: global name 'infj' is not defined [test_contains.py]=1 # Code "while False: yield None" is optimized away in compilation - [test_decorators.py]=1 # Control flow wrt "if elif" + [test_contextlib_async.py]=1 # Investigate + [test_context.py]=1 + [test_coroutines.py]=1 # Parse error + [test_curses.py]=1 # Parse error + [test_cmath.py]=1 # Syntax error - investigate + [test_decorators.py]=1 # Control flow wrt "if elif" + [test_dis.py]=1 # We change line numbers - duh! + # ... + ) + 3.8) + SKIP_TESTS=( + [test_contains.py]=1 # Code "while False: yield None" is optimized away in compilation + [test_decorators.py]=1 # Control flow wrt "if elif" + [test_dis.py]=1 # We change line numbers - duh! + [test_pow.py]=1 # Control flow wrt "continue" + [test_quopri.py]=1 # Only fails on POWER + # ... ) ;; *) diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index 88286465..76b7f705 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -316,10 +316,18 @@ TABLE_DIRECT = { 'compare_chained1': ( '%[3]{pattr.replace("-", " ")} %p %p', (0, 19), (-2, 19)), 'compare_chained2': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)), # 'classdef': (), # handled by n_classdef() + + # A custom rule in n_function def distinguishes whether to call this or + # function_def_async 'function_def': ( '\n\n%|def %c\n', -2), # -2 to handle closures + 'function_def_deco': ( '\n\n%c', 0), 'mkfuncdeco': ( '%|@%c\n%c', 0, 1), + + # A custom rule in n_function def distinguishes whether to call this or + # function_def_async 'mkfuncdeco0': ( '%|def %c\n', 0), + 'classdefdeco': ( '\n\n%c', 0), 'classdefdeco1': ( '%|@%c\n%c', 0, 1), 'kwarg': ( '%[0]{pattr}=%c', 1), # Change when Python 2 does LOAD_STR diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index 3861bce6..70e46889 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -49,7 +49,7 @@ def customize_for_version3(self, version): "raise_stmt2": ("%|raise %c from %c\n", 0, 1), "store_locals": ("%|# inspect.currentframe().f_locals = __locals__\n",), "withstmt": ("%|with %c:\n%+%c%-", 0, 3), - "withasstmt": ("%|with %c as (%c):\n%+%c%-", 0, 2, 3), + "withasstmt": ("%|with %c as %c:\n%+%c%-", 0, 2, 3), } ) diff --git a/uncompyle6/semantics/customize35.py b/uncompyle6/semantics/customize35.py index 7bc48f15..6e2cd770 100644 --- a/uncompyle6/semantics/customize35.py +++ b/uncompyle6/semantics/customize35.py @@ -17,44 +17,47 @@ from xdis.code import iscode from xdis.util import COMPILER_FLAG_BIT -from uncompyle6.semantics.consts import ( - INDENT_PER_LEVEL, TABLE_DIRECT) -from uncompyle6.semantics.helper import ( - flatten_list, gen_function_parens_adjust) +from uncompyle6.semantics.consts import INDENT_PER_LEVEL, TABLE_DIRECT +from uncompyle6.semantics.helper import flatten_list, gen_function_parens_adjust ####################### # Python 3.5+ Changes # ####################### def customize_for_version35(self, version): - TABLE_DIRECT.update({ - 'await_expr': ( 'await %c', 0), - 'await_stmt': ( '%|%c\n', 0), - 'async_for_stmt': ( - '%|async for %c in %c:\n%+%|%c%-\n\n', 9, 1, 25 ), - 'async_forelse_stmt': ( - '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', - 9, 1, 25, (27, 'else_suite') ), - 'async_with_stmt': ( - '%|async with %c:\n%+%|%c%-', - (0, 'expr'), 7 ), - 'async_with_as_stmt': ( - '%|async with %c as %c:\n%+%|%c%-', - (0, 'expr'), (6, 'store'), 7), - 'unmap_dict': ( '{**%C}', (0, -1, ', **') ), - # 'unmapexpr': ( '{**%c}', 0), # done by n_unmapexpr - - }) + TABLE_DIRECT.update( + { + "await_expr": ("await %c", 0), + "await_stmt": ("%|%c\n", 0), + "async_for_stmt": ("%|async for %c in %c:\n%+%|%c%-\n\n", 9, 1, 25), + "async_forelse_stmt": ( + "%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", + 9, + 1, + 25, + (27, "else_suite"), + ), + "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 7), + "async_with_as_stmt": ( + "%|async with %c as %c:\n%+%c%-", + (0, "expr"), + (6, "store"), + 7, + ), + "unmap_dict": ("{**%C}", (0, -1, ", **")), + # "unmapexpr": ( "{**%c}", 0), # done by n_unmapexpr + } + ) def async_call(node): - self.f.write('async ') - node.kind == 'call' + self.f.write("async ") + node.kind == "call" p = self.prec self.prec = 80 - self.template_engine(('%c(%P)', 0, (1, -4, ', ', - 100)), node) + self.template_engine(("%c(%P)", 0, (1, -4, ", ", 100)), node) self.prec = p - node.kind == 'async_call' + node.kind == "async_call" self.prune() + self.n_async_call = async_call def n_build_list_unpack(node): @@ -90,11 +93,13 @@ def customize_for_version35(self, version): if value.startswith("("): assert value.endswith(")") use_star = False - value = value[1:-1].rstrip(" ") # Remove starting '(' and trailing ')' and additional spaces + value = value[1:-1].rstrip( + " " + ) # Remove starting "(" and trailing ")" and additional spaces if value == "": pass else: - if value.endswith(","): # if args has only one item + if value.endswith(","): # if args has only one item value = value[:-1] if line_number != self.line_number: sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1] @@ -114,6 +119,7 @@ def customize_for_version35(self, version): self.prec = p self.prune() return + self.n_build_list_unpack = n_build_list_unpack def n_call(node): @@ -123,7 +129,7 @@ def customize_for_version35(self, version): for i in mapping[1:]: key = key[i] pass - if key.kind.startswith('CALL_FUNCTION_VAR_KW'): + if key.kind.startswith("CALL_FUNCTION_VAR_KW"): # Python 3.5 changes the stack position of # *args: kwargs come after *args whereas # in earlier Pythons, *args is at the end @@ -137,12 +143,12 @@ def customize_for_version35(self, version): kwarg_pos = entry[2][1] args_pos = kwarg_pos - 1 # Put last node[args_pos] after subsequent kwargs - while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node): + while node[kwarg_pos] == "kwarg" and kwarg_pos < len(node): # swap node[args_pos] with node[kwargs_pos] node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos] args_pos = kwarg_pos kwarg_pos += 1 - elif key.kind.startswith('CALL_FUNCTION_VAR'): + elif key.kind.startswith("CALL_FUNCTION_VAR"): # CALL_FUNCTION_VAR's top element of the stack contains # the variable argument list, then comes # annotation args, then keyword args. @@ -153,65 +159,79 @@ def customize_for_version35(self, version): kwargs = (argc >> 8) & 0xFF # FIXME: handle annotation args if nargs > 0: - template = ('%c(%C, ', 0, (1, nargs+1, ', ')) + template = ("%c(%C, ", 0, (1, nargs + 1, ", ")) else: - template = ('%c(', 0) + template = ("%c(", 0) self.template_engine(template, node) - args_node = node[-2] - if args_node in ('pos_arg', 'expr'): + args_node = node[-2] + if args_node in ("pos_arg", "expr"): args_node = args_node[0] - if args_node == 'build_list_unpack': - template = ('*%P)', (0, len(args_node)-1, ', *', 100)) + if args_node == "build_list_unpack": + template = ("*%P)", (0, len(args_node) - 1, ", *", 100)) self.template_engine(template, args_node) else: if len(node) - nargs > 3: - template = ('*%c, %C)', nargs+1, (nargs+kwargs+1, -1, ', ')) + template = ("*%c, %C)", nargs + 1, (nargs + kwargs + 1, -1, ", ")) else: - template = ('*%c)', nargs+1) + template = ("*%c)", nargs + 1) self.template_engine(template, node) self.prune() else: gen_function_parens_adjust(key, node) self.default(node) + self.n_call = n_call - def n_function_def(node): - n0 = node[0] - is_code = False - for i in list(range(len(n0)-2, -1, -1)): - code_node = n0[i] - if hasattr(code_node, 'attr') and iscode(code_node.attr): - is_code = True + def is_async_fn(node): + code_node = node[0][0] + for n in node[0]: + if hasattr(n, "attr") and iscode(n.attr): + code_node = n break + pass + pass - if is_code and ( + is_code = hasattr(code_node, "attr") and iscode(code_node.attr) + return is_code and ( code_node.attr.co_flags & ( COMPILER_FLAG_BIT["COROUTINE"] | COMPILER_FLAG_BIT["ITERABLE_COROUTINE"] | COMPILER_FLAG_BIT["ASYNC_GENERATOR"] ) - ): - self.template_engine(('\n\n%|async def %c\n', - -2), node) + ) + + def n_function_def(node): + if is_async_fn(node): + self.template_engine(("\n\n%|async def %c\n", -2), node) else: - self.template_engine(('\n\n%|def %c\n', -2), - node) + self.default(node) self.prune() + self.n_function_def = n_function_def + def n_mkfuncdeco0(node): + if is_async_fn(node): + self.template_engine(("%|async def %c\n", 0), node) + else: + self.default(node) + self.prune() + + self.n_mkfuncdeco0 = n_mkfuncdeco0 + def unmapexpr(node): last_n = node[0][-1] for n in node[0]: self.preorder(n) if n != last_n: - self.f.write(', **') + self.f.write(", **") pass pass self.prune() pass + self.n_unmapexpr = unmapexpr # FIXME: start here @@ -228,67 +248,75 @@ def customize_for_version35(self, version): # then the first * has already been printed. # Until I have a better way to check for CALL_FUNCTION_VAR, # will assume that if the text ends in *. - last_was_star = self.f.getvalue().endswith('*') + last_was_star = self.f.getvalue().endswith("*") - if lastnodetype.startswith('BUILD_LIST'): - self.write('['); endchar = ']' - elif lastnodetype.startswith('BUILD_TUPLE'): + if lastnodetype.startswith("BUILD_LIST"): + self.write("[") + endchar = "]" + elif lastnodetype.startswith("BUILD_TUPLE"): # Tuples can appear places that can NOT # have parenthesis around them, like array # subscripts. We check for that by seeing # if a tuple item is some sort of slice. no_parens = False for n in node: - if n == 'expr' and n[0].kind.startswith('build_slice'): + if n == "expr" and n[0].kind.startswith("build_slice"): no_parens = True break pass if no_parens: - endchar = '' + endchar = "" else: - self.write('('); endchar = ')' + self.write("(") + endchar = ")" pass - elif lastnodetype.startswith('BUILD_SET'): - self.write('{'); endchar = '}' - elif lastnodetype.startswith('BUILD_MAP_UNPACK'): - self.write('{*'); endchar = '}' - elif lastnodetype.startswith('ROT_TWO'): - self.write('('); endchar = ')' + elif lastnodetype.startswith("BUILD_SET"): + self.write("{") + endchar = "}" + elif lastnodetype.startswith("BUILD_MAP_UNPACK"): + self.write("{*") + endchar = "}" + elif lastnodetype.startswith("ROT_TWO"): + self.write("(") + endchar = ")" else: - raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack') + raise TypeError( + "Internal Error: n_build_list expects list, tuple, set, or unpack" + ) flat_elems = flatten_list(node) self.indent_more(INDENT_PER_LEVEL) - sep = '' + sep = "" for elem in flat_elems: - if elem in ('ROT_THREE', 'EXTENDED_ARG'): + if elem in ("ROT_THREE", "EXTENDED_ARG"): continue - assert elem == 'expr' + assert elem == "expr" line_number = self.line_number value = self.traverse(elem) - if elem[0] == 'tuple': - assert value[0] == '(' - assert value[-1] == ')' + if elem[0] == "tuple": + assert value[0] == "(" + assert value[-1] == ")" value = value[1:-1] - if value[-1] == ',': + if value[-1] == ",": # singleton tuple value = value[:-1] else: - value = '*' + value + value = "*" + value if line_number != self.line_number: - sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1] + sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1] else: - if sep != '': sep += ' ' + if sep != "": + sep += " " if not last_was_star: pass else: last_was_star = False self.write(sep, value) - sep = ',' - if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'): - self.write(',') + sep = "," + if lastnode.attr == 1 and lastnodetype.startswith("BUILD_TUPLE"): + self.write(",") self.write(endchar) self.indent_less(INDENT_PER_LEVEL)