From d0a98bdbc64ae8faef4e7b2bead61b25d4aa6312 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 4 Nov 2022 00:42:50 -0400 Subject: [PATCH 1/3] Correct 3.0 list comprehension parsing --- uncompyle6/parsers/parse30.py | 7 ++++++- uncompyle6/semantics/gencomp.py | 21 ++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 79ecbe73..bf8815bf 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -85,6 +85,11 @@ class Python30Parser(Python31Parser): LOAD_FAST FOR_ITER store comp_iter JUMP_BACK _come_froms POP_TOP JUMP_BACK + list_for ::= DUP_TOP STORE_FAST + expr_or_arg + FOR_ITER + store list_iter jb_or_c + set_comp ::= set_comp_header LOAD_FAST FOR_ITER store comp_iter JUMP_BACK @@ -219,7 +224,7 @@ class Python30Parser(Python31Parser): # lc_body ::= LOAD_NAME expr LIST_APPEND # lc_body ::= expr LIST_APPEND # list_comp ::= BUILD_LIST_0 list_iter - # list_for ::= expr FOR_ITER store list_iter jb_or_c + list_for ::= expr FOR_ITER store list_iter jb_or_c # list_if ::= expr jmp_false list_iter # list_if ::= expr jmp_false_then list_iter # list_if_not ::= expr jmp_true list_iter diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index ae3f82eb..cf2278ce 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -384,7 +384,7 @@ class ComprehensionMixin: while n in ("list_iter", "list_afor", "list_afor2", "comp_iter"): # iterate one nesting deeper - if self.version == 3.0 and len(n) == 3: + if self.version == (3, 0) and len(n) == 3: assert n[0] == "expr" and n[1] == "expr" n = n[1] elif n == "list_afor": @@ -397,11 +397,17 @@ class ComprehensionMixin: n = n[0] if n in ("list_for", "comp_for"): - if n[2] == "store" and not store: - store = n[2] + n_index = 3 + if ((n[2] == "store") + or (self.version == (3, 0) and n[4] == "store") and not store): + if self.version == (3, 0): + store = n[4] + n_index = 5 + else: + store = n[2] if not comp_store: comp_store = store - n = n[3] + n = n[n_index] elif n in ("list_if", "list_if_not", "list_if37", "list_if37_not", "comp_if", "comp_if_not"): @@ -443,7 +449,11 @@ class ComprehensionMixin: self.write(": ") self.preorder(n[1]) else: - self.preorder(n[0]) + if self.version == (3, 0): + body = n[1] + else: + body = n[0] + self.preorder(body) if node == "list_comp_async": self.write(" async") @@ -455,6 +465,7 @@ class ComprehensionMixin: if comp_store: self.preorder(comp_store) + comp_store = None else: self.preorder(store) From 0be3fc657bded4c61fc6692c1dada53a54f4a2de Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 4 Nov 2022 00:48:53 -0400 Subject: [PATCH 2/3] Sync with 3.3-3.5 branch --- uncompyle6/semantics/gencomp.py | 43 ++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/uncompyle6/semantics/gencomp.py b/uncompyle6/semantics/gencomp.py index cf2278ce..4a3e4f00 100644 --- a/uncompyle6/semantics/gencomp.py +++ b/uncompyle6/semantics/gencomp.py @@ -27,6 +27,7 @@ from uncompyle6.semantics.consts import PRECEDENCE from uncompyle6.semantics.helper import is_lambda_mode from uncompyle6.scanners.tok import Token + class ComprehensionMixin: """ These functions hand nonterminal common actions that occur @@ -37,12 +38,13 @@ class ComprehensionMixin: Python source code. In source code, the implicit function calls are not seen. """ + def closure_walk(self, node, collection_index): """ Dictionary and comprehensions using closure the way they are done in Python3. """ p = self.prec - self.prec = 27 + self.prec = PRECEDENCE["lambda_body"] - 1 code_index = 0 if node[0] == "load_genexpr" else 1 tree = self.get_comprehension_function(node, code_index=code_index) @@ -114,7 +116,10 @@ class ComprehensionMixin: elif node[0] == "load_closure": cn = node[1] - elif self.version >= (3, 0) and node in ("generator_exp", "generator_exp_async"): + elif self.version >= (3, 0) and node in ( + "generator_exp", + "generator_exp_async", + ): if node[0] == "load_genexpr": load_genexpr = node[0] elif node[1] == "load_genexpr": @@ -143,7 +148,9 @@ class ComprehensionMixin: if is_lambda_mode(self.compile_mode): p_save = self.p self.p = get_python_parser( - self.version, compile_mode="exec", is_pypy=self.is_pypy, + self.version, + compile_mode="exec", + is_pypy=self.is_pypy, ) tree = self.build_ast(code._tokens, code._customize, code) self.p = p_save @@ -398,8 +405,11 @@ class ComprehensionMixin: if n in ("list_for", "comp_for"): n_index = 3 - if ((n[2] == "store") - or (self.version == (3, 0) and n[4] == "store") and not store): + if ( + (n[2] == "store") + or (self.version == (3, 0) and n[4] == "store") + and not store + ): if self.version == (3, 0): store = n[4] n_index = 5 @@ -408,9 +418,14 @@ class ComprehensionMixin: if not comp_store: comp_store = store n = n[n_index] - elif n in ("list_if", "list_if_not", - "list_if37", "list_if37_not", - "comp_if", "comp_if_not"): + elif n in ( + "list_if", + "list_if_not", + "list_if37", + "list_if37_not", + "comp_if", + "comp_if_not", + ): have_not = n in ("list_if_not", "comp_if_not", "list_if37_not") if n in ("list_if37", "list_if37_not"): n = n[1] @@ -492,7 +507,7 @@ class ComprehensionMixin: if have_not: self.write("not ") pass - self.prec = 27 + self.prec = PRECEDENCE["lambda_body"] - 1 self.preorder(if_node) pass self.prec = p @@ -503,7 +518,7 @@ class ComprehensionMixin: find the comprehension node buried in the tree which may be surrounded with start-like symbols or dominiators,. """ - self.prec = 27 + self.prec = PRECEDENCE["lambda_body"] - 1 code_node = node[code_index] if code_node == "load_genexpr": code_node = code_node[0] @@ -519,7 +534,9 @@ class ComprehensionMixin: if self.compile_mode in ("listcomp",): # add other comprehensions to this list p_save = self.p self.p = get_python_parser( - self.version, compile_mode="exec", is_pypy=self.is_pypy, + self.version, + compile_mode="exec", + is_pypy=self.is_pypy, ) tree = self.build_ast( code._tokens, code._customize, code, is_lambda=self.is_lambda @@ -538,9 +555,7 @@ class ComprehensionMixin: if tree[0] in ("dom_start", "dom_start_opt"): tree = tree[1] - while len(tree) == 1 or ( - tree in ("stmt", "sstmt", "return", "return_expr") - ): + while len(tree) == 1 or (tree in ("stmt", "sstmt", "return", "return_expr")): self.prec = 100 tree = tree[1] if tree[0] in ("dom_start", "dom_start_opt") else tree[0] return tree From 766618ba48fc8d87a81fbb8d3118926565b4dfb1 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 4 Nov 2022 00:53:45 -0400 Subject: [PATCH 3/3] Alow 3.0 setup --- ....1-3.2-versions => pyenv-3.0-3.2-versions} | 2 +- admin-tools/setup-python-3.0.sh | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) rename admin-tools/{pyenv-3.1-3.2-versions => pyenv-3.0-3.2-versions} (86%) create mode 100644 admin-tools/setup-python-3.0.sh diff --git a/admin-tools/pyenv-3.1-3.2-versions b/admin-tools/pyenv-3.0-3.2-versions similarity index 86% rename from admin-tools/pyenv-3.1-3.2-versions rename to admin-tools/pyenv-3.0-3.2-versions index 334a2631..df84c5bf 100644 --- a/admin-tools/pyenv-3.1-3.2-versions +++ b/admin-tools/pyenv-3.0-3.2-versions @@ -6,4 +6,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then echo "This script should be *sourced* rather than run directly through bash" exit 1 fi -export PYVERSIONS='3.1.5 3.2.6' +export PYVERSIONS='3.0.1 3.1.5 3.2.6' diff --git a/admin-tools/setup-python-3.0.sh b/admin-tools/setup-python-3.0.sh new file mode 100644 index 00000000..27282d53 --- /dev/null +++ b/admin-tools/setup-python-3.0.sh @@ -0,0 +1,35 @@ +#!/bin/bash +PYTHON_VERSION=3.0.1 +pyenv local $PYTHON_VERSION + +# FIXME put some of the below in a common routine +function checkout_version { + local repo=$1 + version=${2:-python-3.0-to-3.2} + echo Checking out $version on $repo ... + (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \ + git pull + return $? +} + +function finish { + cd $owd +} + +export PATH=$HOME/.pyenv/bin/pyenv:$PATH +owd=$(pwd) +bs=${BASH_SOURCE[0]} +if [[ $0 == $bs ]] ; then + echo "This script should be *sourced* rather than run directly through bash" + exit 1 +fi + +mydir=$(dirname $bs) +fulldir=$(readlink -f $mydir) +cd $fulldir/.. +(cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis && + checkout_version python-uncompyle6) +cd $owd +rm -v */.python-version || true + +git checkout python-3.0-to-3.2 && git pull && pyenv local $PYTHON_VERSION