From a68f440d6faa6abc3b318a0097024e95eab9744a Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 17 Sep 2022 01:59:42 -0400 Subject: [PATCH] Sync with decompyle3 Better PyPy 3.7 tolerance --- test/stdlib/runtests.sh | 27 +++++++- uncompyle6/parsers/parse37.py | 15 ++--- uncompyle6/semantics/customize.py | 97 +++++++++++++++++++---------- uncompyle6/semantics/customize36.py | 2 + uncompyle6/semantics/gencomp.py | 23 ++++++- 5 files changed, 119 insertions(+), 45 deletions(-) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh index 01891309..b0c74644 100755 --- a/test/stdlib/runtests.sh +++ b/test/stdlib/runtests.sh @@ -29,7 +29,19 @@ function displaytime { # Python version setup FULLVERSION=$(pyenv local) PYVERSION=${FULLVERSION%.*} -MINOR=${FULLVERSION##?.?.} + +if [[ $PYVERSION =~ 'pypy' ]] ; then + IS_PYPY=1 +else + IS_PYPY=0 +fi + +if [[ $FULLVERSION =~ pypy([2-3])\.([7-9]) ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" +else + MINOR=${FULLVERSION##?.?.} +fi STOP_ONERROR=${STOP_ONERROR:-1} @@ -150,7 +162,12 @@ done mkdir $TESTDIR || exit $? -cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR + +if ((IS_PYPY)); then + cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib-python/${MAJOR}/test $TESTDIR +else + cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR +fi if [[ $PYVERSION == 3.2 ]] ; then cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR cd $TESTDIR @@ -209,7 +226,11 @@ for file in $files; do ((i++)) # (( i > 40 )) && break short_name=$(basename $file .py) - decompiled_file=$short_name-${PYVERSION}.pyc + if ((IS_PYPY)); then + decompiled_file=$short_name-${MAJOR}.${MINOR}.pyc + else + decompiled_file=$short_name-${PYVERSION}.pyc + fi $fulldir/compile-file.py $file && \ mv $file{,.orig} && \ echo ========== $(date +%X) Decompiling $file =========== diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index 5d168ab4..80d9c6dd 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -138,7 +138,8 @@ class Python37Parser(Python37BaseParser): returns ::= _stmts return stmt ::= genexpr_func - genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter JUMP_BACK + genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter + _come_froms JUMP_BACK _come_froms """ pass @@ -1365,7 +1366,7 @@ class Python37Parser(Python37BaseParser): genexpr_func_async ::= LOAD_ARG func_async_prefix store func_async_middle comp_iter - JUMP_LOOP COME_FROM + JUMP_BACK COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP # FIXME this is a workaround for probalby some bug in the Earley parser @@ -1377,7 +1378,7 @@ class Python37Parser(Python37BaseParser): list_afor2 ::= func_async_prefix store func_async_middle list_iter - JUMP_LOOP COME_FROM + JUMP_BACK COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP list_comp_async ::= BUILD_LIST_0 LOAD_ARG list_afor2 @@ -1463,13 +1464,13 @@ class Python37Parser(Python37BaseParser): genexpr_func_async ::= LOAD_ARG async_iter store_async_iter_end comp_iter - JUMP_LOOP COME_FROM + JUMP_BACK COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP list_afor2 ::= async_iter store list_iter - JUMP_LOOP + JUMP_BACK COME_FROM_FINALLY END_ASYNC_FOR @@ -1479,7 +1480,7 @@ class Python37Parser(Python37BaseParser): store func_async_middle set_iter - JUMP_LOOP COME_FROM + JUMP_BACK COME_FROM POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP set_afor2 ::= expr_or_arg @@ -1490,7 +1491,7 @@ class Python37Parser(Python37BaseParser): set_iter_async ::= async_iter store set_iter - JUMP_LOOP + JUMP_BACK _come_froms END_ASYNC_FOR diff --git a/uncompyle6/semantics/customize.py b/uncompyle6/semantics/customize.py index 43942937..93855726 100644 --- a/uncompyle6/semantics/customize.py +++ b/uncompyle6/semantics/customize.py @@ -17,7 +17,13 @@ """ from uncompyle6.parsers.treenode import SyntaxTree -from uncompyle6.semantics.consts import INDENT_PER_LEVEL, NO_PARENTHESIS_EVER, PRECEDENCE, TABLE_R, TABLE_DIRECT +from uncompyle6.semantics.consts import ( + INDENT_PER_LEVEL, + NO_PARENTHESIS_EVER, + PRECEDENCE, + TABLE_R, + TABLE_DIRECT, +) from uncompyle6.semantics.helper import flatten_list from uncompyle6.scanners.tok import Token @@ -27,22 +33,29 @@ def customize_for_version(self, is_pypy, version): ######################## # PyPy changes ####################### + # fmt: off TABLE_DIRECT.update({ - "assert": ("%|assert %c\n", 0), - "assert_pypy": ( '%|assert %c\n' , (1, 'assert_expr') ), - # This is as a result of an if transformation - 'assert0_pypy': ( '%|assert %c\n' , 0), - 'assert_not_pypy': ( '%|assert not %c\n' , (1, 'assert_exp') ), - 'assert2_not_pypy': ( '%|assert not %c, %c\n' , (1, 'assert_exp'), - (4, 'expr') ), - 'assert2_pypy': ( '%|assert %c, %c\n' , (1, 'assert_expr'), - (4, 'expr') ), - 'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ), - 'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ), - 'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ), - 'assign2_pypy': ( '%|%c, %c = %c, %c\n', 3, 2, 0, 1), + "assert": ("%|assert %c\n", 0), + # This can happen as a result of an if transformation + "assert2": ("%|assert %c, %c\n", 0, 3), + "assert_pypy": ( "%|assert %c\n" , (1, "assert_expr") ), + # This is as a result of an if transformation + 'assert0_pypy': ( "%|assert %c\n" , 0), + + 'assert_not_pypy': ( "%|assert not %c\n" , (1, "assert_exp") ), + "assert2_not_pypy": ( + "%|assert not %c, %c\n", + (1, "assert_exp"), + (4, "expr"), + ), + + "try_except_pypy": ( "%|try:\n%+%c%-%c\n\n", 1, 2 ), + "tryfinallystmt_pypy": ( "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", 1, 3 ), + "assign3_pypy": ( "%|%c, %c, %c = %c, %c, %c\n", 5, 4, 3, 0, 1, 2 ), + "assign2_pypy": ( "%|%c, %c = %c, %c\n", 3, 2, 0, 1), }) + # fmt: on if version[:2] >= (3, 7): @@ -53,12 +66,18 @@ def customize_for_version(self, is_pypy, version): kw_names = node[-2] assert kw_names == "pypy_kw_keys" - flat_elems = flatten_list(node[1:-2]) - n = len(flat_elems) - assert n == arg_count kwargs_names = kw_names[0].attr kwarg_count = len(kwargs_names) pos_argc = arg_count - kwarg_count + + flat_elems = flatten_list(node[1:-2]) + n = len(flat_elems) + assert n == arg_count, "n: %s, arg_count: %s\n%s" % ( + n, + arg_count, + node, + ) + sep = "" for i in range(pos_argc): @@ -95,20 +114,23 @@ def customize_for_version(self, is_pypy, version): ######################## # Without PyPy ####################### - TABLE_DIRECT.update({ - # "assert" and "assert_expr" are added via transform rules. - "assert": ("%|assert %c\n", 0), - "assert2": ("%|assert %c, %c\n", 0, 3), - - # Created only via transformation - "assertnot": ("%|assert not %p\n", (0, PRECEDENCE['unary_not'])), - "assert2not": ( "%|assert not %p, %c\n" , - (0, PRECEDENCE['unary_not']), 3 ), - - "assign2": ("%|%c, %c = %c, %c\n", 3, 4, 0, 1), - "assign3": ("%|%c, %c, %c = %c, %c, %c\n", 5, 6, 7, 0, 1, 2), - "try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3), - }) + TABLE_DIRECT.update( + { + # "assert" and "assert_expr" are added via transform rules. + "assert": ("%|assert %c\n", 0), + "assert2": ("%|assert %c, %c\n", 0, 3), + # Created only via transformation + "assertnot": ("%|assert not %p\n", (0, PRECEDENCE["unary_not"])), + "assert2not": ( + "%|assert not %p, %c\n", + (0, PRECEDENCE["unary_not"]), + 3, + ), + "assign2": ("%|%c, %c = %c, %c\n", 3, 4, 0, 1), + "assign3": ("%|%c, %c, %c = %c, %c, %c\n", 5, 6, 7, 0, 1, 2), + "try_except": ("%|try:\n%+%c%-%c\n\n", 1, 3), + } + ) if version >= (3, 0): if version >= (3, 2): TABLE_DIRECT.update( @@ -187,6 +209,7 @@ def customize_for_version(self, is_pypy, version): } ) if version == (2, 4): + def n_iftrue_stmt24(node): self.template_engine(("%c", 0), node) self.default(node) @@ -195,6 +218,7 @@ def customize_for_version(self, is_pypy, version): self.n_iftrue_stmt24 = n_iftrue_stmt24 elif version < (1, 4): from uncompyle6.semantics.customize14 import customize_for_version14 + customize_for_version14(self, version) def n_call(node): @@ -210,9 +234,16 @@ def customize_for_version(self, is_pypy, version): sep = ", " self.write(")") else: - self.template_engine(("%p(%P)", - (0, "expr", 100), (1,-1,", ", NO_PARENTHESIS_EVER)), node) + self.template_engine( + ( + "%p(%P)", + (0, "expr", 100), + (1, -1, ", ", NO_PARENTHESIS_EVER), + ), + node, + ) self.prune() + self.n_call = n_call else: # 1.0 <= version <= 2.3: diff --git a/uncompyle6/semantics/customize36.py b/uncompyle6/semantics/customize36.py index dde3dc3b..6c2339c7 100644 --- a/uncompyle6/semantics/customize36.py +++ b/uncompyle6/semantics/customize36.py @@ -167,6 +167,8 @@ def customize_for_version36(self, version): if node == "classdefdeco2": if isinstance(node[1][1].attr, str): class_name = node[1][1].attr + if self.is_pypy and class_name.find("") > 0: + class_name = class_name.split(".")[-1] else: class_name = node[1][2].attr build_class = node diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index 52f228ca..ae3f82eb 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -97,9 +97,12 @@ class ComprehensionMixin: self.prec = p def comprehension_walk( - self, node, iter_index: Optional[int], code_index: int = -5, + self, + node, + iter_index: Optional[int], + code_index: int = -5, ): - p = self.prec + p: int = self.prec self.prec = PRECEDENCE["lambda_body"] - 1 # FIXME: clean this up @@ -152,6 +155,22 @@ class ComprehensionMixin: while len(tree) == 1: tree = tree[0] + if tree == "stmts": + # FIXME: rest is a return None? + # Verify this + # rest = tree[1:] + tree = tree[0] + elif tree == "lambda_start": + assert len(tree) <= 3 + tree = tree[-2] + if tree == "return_expr_lambda": + tree = tree[1] + pass + + if tree in ("genexpr_func_async",): + if tree[3] == "comp_iter": + iter_index = 3 + n = tree[iter_index] assert n == "comp_iter", n.kind