jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 09:22:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'python-3.0-to-3.2' into python-2.4-to-2.7

Browse Source
This commit is contained in:
rocky
2024-03-02 05:31:10 -05:00
parent 6c26f3fb8d 371b5c7600
commit 0993bb026b
23 changed files with 413 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/FUNDING.yml vendored
View File

@@ -6,7 +6,7 @@ open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
liberapay: rocky
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

26
admin-tools/check-3.0-3.2-versions.sh Normal file
View File

@@ -0,0 +1,26 @@
#!/bin/bash
# Run tests over all Python versions in branch python-3.0-3.2
owd=$(pwd)
cd $(dirname ${BASH_SOURCE[0]})
if ! source ./pyenv-3.0-3.2-versions ; then
exit $?
fi
if ! source ./setup-python-3.0.sh ; then
exit $?
fi
cd ..
for version in $PYVERSIONS; do
echo --- $version ---
if ! pyenv local $version ; then
exit $?
fi
make clean && python setup.py develop
if ! make check ; then
exit $?
fi
echo === $version ===
done
cd $owd

2
admin-tools/merge-for-2.4.sh
View File

@@ -1,5 +1,7 @@
#/bin/bash
owd=$(pwd)
cd $(dirname ${BASH_SOURCE[0]})
if . ./setup-python-2.4.sh; then
git merge python-3.0-to-3.2
fi
cd $owd

2
admin-tools/merge-for-3.0.sh
View File

@@ -1,5 +1,7 @@
#/bin/bash
owd=$(pwd)
cd $(dirname ${BASH_SOURCE[0]})
if . ./setup-python-3.0.sh; then
git merge python-3.3-to-3.5
fi
cd $owd

2
admin-tools/merge-for-3.3.sh
View File

@@ -1,5 +1,7 @@
#/bin/bash
owd=$(pwd)
cd $(dirname ${BASH_SOURCE[0]})
if . ./setup-python-3.3.sh; then
git merge master
fi
cd $owd

34
test/stdlib/3.2-exclude.sh
View File

@@ -1,7 +1,14 @@
SKIP_TESTS=(
[test_descr.py]=1 # FIXME: Works on c90ff51?
[test_descr.py]=1
# [test_descr.py]=pytest_module # FIXME: Works on c90ff51?
# AssertionError: 'D(4)C(4)A(4)' != 'D(4)C(4)B(4)A(4)'
# - D(4)C(4)A(4)
# + D(4)C(4)B(4)A(4)
# ? ++++
[test_cmath.py]=1 # FIXME
[test_cmath.py]=1 # Control-flow "elif else -> else: if else"
# [test_cmath.py]=pytest_module
# AssertionError: rect1000: rect(complex(0.0, 0.0))
# Expected: complex(0.0, 0.0)
# Received: complex(0.0, -1.0)
@@ -9,14 +16,27 @@ SKIP_TESTS=(
[test_cmd_line.py]=1
[test_collections.py]=1
[test_collections.py]=1 # fail on its own
# E TypeError: __new__() takes exactly 4 arguments (1 given)
[test_concurrent_futures.py]=1 # too long to run over 46 seconds by itself
[test_datetimetester.py]=1
[test_decimal.py]=1
[test_dictcomps.py]=1 # FIXME: semantic error: actual = {k:v for k in }
[test_doctest.py]=1 # test failures
[test_datetime.py]=pytest_module
[test_decimal.py]=1 # Fails on its own, even with pytest
[test_dictcomps.py]=1
# [test_dictcomps.py]=pytest_module # FIXME: semantic error: actual = {k:v for k in }
# assert (count * 2) <= i
[test_doctest.py]=1 # Missing pytest fixture
# [test_doctest.py]=pytest_module
# fixture 'coverdir' not found
[test_dis.py]=1 # We change line numbers - duh!
[test_exceptions.py]=1 # parse error
# [test_exceptions.py]=pytest_module # parse error
[test_modulefinder.py]=1 # test failures
[test_multiprocessing.py]=1 # test takes too long to run: 35 seconds

2
test/stdlib/3.3-exclude.sh
View File

@@ -10,7 +10,7 @@ SKIP_TESTS=(
# tgt.append(elem)
[test_itertools.py]=1
[test_buffer.py]=1 # FIXME: Works on c90ff51
[test_buffer.py]=pytest # FIXME: Works on c90ff51
[test_cmath.py]=pytest
[test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines

9
test/stdlib/3.8-exclude.sh
View File

@@ -28,12 +28,12 @@ SKIP_TESTS=(
# These and the above may be due to new code generation or tests
# between 3.8.3 and 3.8.5 ?
[test_decorators.py]=1 #
[test_decorators.py]=1 # parse error
[test_dtrace.py]=1 #
[test_exceptions.py]=1 #
[test_dtrace.py]=1 # parse error
[test_exceptions.py]=1 # parse error
[test_ftplib.py]=1 #
[test_gc.py]=1 #
[test_gc.py]=1 # FIXME: return return strip_python_stderr(stderr)
[test_gzip.py]=1 #
[test_hashlib.py]=1 #
[test_iter.py]=1 #
@@ -51,7 +51,6 @@ SKIP_TESTS=(
[test_audioop.py]=1 # test failure
[test_audit.py]=1 # parse error
[test_base64.py]=1 # parse error
[test_baseexception.py]=1 #
[test_bigaddrspace.py]=1 # parse error
[test_bigmem.py]=1 # parse error

12
test/stdlib/runtests.sh
View File

@@ -172,12 +172,8 @@ if ((IS_PYPY)); then
else
cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
fi
if [[ $PYVERSION == 3.2 ]] ; then
cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR
cd $TESTDIR
else
cd $TESTDIR/test
fi
cd $TESTDIR/test
pyenv local $FULLVERSION
export PYTHONPATH=$TESTDIR
export PATH=${PYENV_ROOT}/shims:${PATH}
@@ -192,7 +188,7 @@ if [[ -n $1 ]] ; then
typeset -a files_ary=( $(echo $@) )
if (( ${#files_ary[@]} == 1 || DONT_SKIP_TESTS == 1 )) ; then
for file in $files; do
if (( SKIP_TESTS[$file] != "pytest" )); then
if (( SKIP_TESTS[$file] != "pytest" || SKIP_TESTS[$file] != "pytest_module" )); then
SKIP_TESTS[$file]=1;
fi
done
@@ -212,6 +208,8 @@ for file in $files; do
if [[ ${SKIP_TESTS[$file]} == "pytest" ]]; then
PYTHON=pytest
elif [[ ${SKIP_TESTS[$file]} == "pytest_module" ]]; then
PYTHON="$PYTHON -m pytest"
else
if [[ ${SKIP_TESTS[$file]}s == ${NOT_INVERTED_TESTS} ]] ; then
((skipped++))

11
uncompyle6/parsers/parse2.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2022 Rocky Bernstein
# Copyright (c) 2015-2021, 2024 Rocky Bernstein
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
#
# Copyright (c) 1999 John Aycock
@@ -28,8 +28,14 @@ that a later phase can turn into a sequence of ASCII text.
from uncompyle6.parsers.reducecheck import (except_handler_else, ifelsestmt, tryelsestmt)
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.treenode import SyntaxTree
from __future__ import print_function
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
from uncompyle6.parsers.treenode import SyntaxTree
class Python2Parser(PythonParser):
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -403,7 +409,6 @@ class Python2Parser(PythonParser):
"CALL_FUNCTION_VAR_KW",
"CALL_FUNCTION_KW",
):
args_pos, args_kw = self.get_pos_kw(token)
# number of apply equiv arguments:
@@ -524,7 +529,7 @@ class Python2Parser(PythonParser):
custom_seen_ops.add(opname)
continue
elif opname == "LOAD_LISTCOMP":
self.addRule("expr ::= listcomp", nop_func)
self.addRule("expr ::= list_comp", nop_func)
custom_seen_ops.add(opname)
continue
elif opname == "LOAD_SETCOMP":

12
uncompyle6/parsers/parse3.py
View File

@@ -1068,7 +1068,9 @@ class Python3Parser(PythonParser):
)
custom_ops_processed.add(opname)
elif opname == "LOAD_LISTCOMP":
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
self.add_unique_rule(
"expr ::= list_comp", opname, token.attr, customize
)
custom_ops_processed.add(opname)
elif opname == "LOAD_SETCOMP":
# Should this be generalized and put under MAKE_FUNCTION?
@@ -1137,7 +1139,7 @@ class Python3Parser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly
rule_pat = (
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * pos_args_count, opname)
)
@@ -1331,14 +1333,14 @@ class Python3Parser(PythonParser):
# 'exprs' in the rule above into a
# tuple.
rule_pat = (
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
"list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
"expr GET_ITER CALL_FUNCTION_1" % (opname,)
)
self.add_make_function_rule(
rule_pat, opname, token.attr, customize
)
rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname)
)
@@ -1382,7 +1384,7 @@ class Python3Parser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly
rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1"
% ("expr " * pos_args_count, opname)
)

12
uncompyle6/parsers/parse37base.py
View File

@@ -723,7 +723,9 @@ class Python37BaseParser(PythonParser):
)
custom_ops_processed.add(opname)
elif opname == "LOAD_LISTCOMP":
self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
self.add_unique_rule(
"expr ::= list_comp", opname, token.attr, customize
)
custom_ops_processed.add(opname)
elif opname == "LOAD_NAME":
if (
@@ -802,7 +804,7 @@ class Python37BaseParser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly
rule_pat = (
"listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1"
% ("pos_arg " * args_pos, opname)
)
@@ -900,14 +902,14 @@ class Python37BaseParser(PythonParser):
# 'exprs' in the rule above into a
# tuple.
rule_pat = (
"listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
"list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
"expr GET_ITER CALL_FUNCTION_1" % (opname,)
)
self.add_make_function_rule(
rule_pat, opname, token.attr, customize
)
rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
)
self.add_make_function_rule(
@@ -941,7 +943,7 @@ class Python37BaseParser(PythonParser):
# and have GET_ITER CALL_FUNCTION_1
# Todo: For Pypy we need to modify this slightly
rule_pat = (
"listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
"list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
"GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
)
self.add_make_function_rule(

506
uncompyle6/parsers/parse38.py
View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2020, 2022-2023 Rocky Bernstein
# Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -16,269 +16,272 @@
spark grammar differences over Python 3.7 for Python 3.8
"""
from uncompyle6.parser import PythonParserSingle, nop_func
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse37 import Python37Parser
from uncompyle6.parsers.reducecheck.pop_return import pop_return_check
class Python38Parser(Python37Parser):
def p_38_stmt(self, args):
"""
stmt ::= async_for_stmt38
stmt ::= async_forelse_stmt38
stmt ::= call_stmt
stmt ::= continue
stmt ::= for38
stmt ::= forelselaststmt38
stmt ::= forelselaststmtl38
stmt ::= forelsestmt38
stmt ::= try_elsestmtl38
stmt ::= try_except38
stmt ::= try_except38r
stmt ::= try_except38r2
stmt ::= try_except38r3
stmt ::= try_except38r4
stmt ::= try_except_as
stmt ::= try_except_ret38
stmt ::= tryfinally38astmt
stmt ::= tryfinally38rstmt
stmt ::= tryfinally38rstmt2
stmt ::= tryfinally38rstmt3
stmt ::= tryfinally38stmt
stmt ::= whileTruestmt38
stmt ::= whilestmt38
stmt ::= async_for_stmt38
stmt ::= async_forelse_stmt38
stmt ::= call_stmt
stmt ::= continue
stmt ::= for38
stmt ::= forelselaststmt38
stmt ::= forelselaststmtl38
stmt ::= forelsestmt38
stmt ::= try_elsestmtl38
stmt ::= try_except38
stmt ::= try_except38r
stmt ::= try_except38r2
stmt ::= try_except38r3
stmt ::= try_except38r4
stmt ::= try_except_as
stmt ::= try_except_ret38
stmt ::= tryfinally38astmt
stmt ::= tryfinally38rstmt
stmt ::= tryfinally38rstmt2
stmt ::= tryfinally38rstmt3
stmt ::= tryfinally38stmt
stmt ::= whileTruestmt38
stmt ::= whilestmt38
call_stmt ::= call
break ::= POP_BLOCK BREAK_LOOP
break ::= POP_BLOCK POP_TOP BREAK_LOOP
break ::= POP_TOP BREAK_LOOP
break ::= POP_EXCEPT BREAK_LOOP
call_stmt ::= call
break ::= POP_BLOCK BREAK_LOOP
break ::= POP_BLOCK POP_TOP BREAK_LOOP
break ::= POP_TOP BREAK_LOOP
break ::= POP_EXCEPT BREAK_LOOP
# The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed.
# Inside an loop we can have this, which can only appear in side a try/except
# And it can also appear at the end of the try except.
continue ::= POP_EXCEPT JUMP_BACK
# The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed.
# Inside an loop we can have this, which can only appear in side a try/except
# And it can also appear at the end of the try except.
continue ::= POP_EXCEPT JUMP_BACK
# FIXME: this should be restricted to being inside a try block
stmt ::= except_ret38
stmt ::= except_ret38a
# FIXME: this should be restricted to being inside a try block
stmt ::= except_ret38
stmt ::= except_ret38a
# FIXME: this should be added only when seeing GET_AITER or YIELD_FROM
async_for ::= GET_AITER _come_froms
SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
async_for_stmt38 ::= expr async_for
store for_block
COME_FROM_FINALLY
END_ASYNC_FOR
# FIXME: this should be added only when seeing GET_AITER or YIELD_FROM
async_for ::= GET_AITER _come_froms
SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
async_for_stmt38 ::= expr async_for
store for_block
COME_FROM_FINALLY
END_ASYNC_FOR
genexpr_func_async ::= LOAD_ARG func_async_prefix
store comp_iter
JUMP_BACK COME_FROM_FINALLY
END_ASYNC_FOR
genexpr_func_async ::= LOAD_ARG func_async_prefix
store comp_iter
JUMP_BACK COME_FROM_FINALLY
END_ASYNC_FOR
# FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of
# for / forelse is used. Add come froms and check of add up control-flow detection phase.
async_forelse_stmt38 ::= expr
GET_AITER
SETUP_FINALLY
GET_ANEXT
LOAD_CONST
YIELD_FROM
POP_BLOCK
store for_block
COME_FROM_FINALLY
END_ASYNC_FOR
else_suite
# Seems to be used to discard values before a return in a "for" loop
discard_top ::= ROT_TWO POP_TOP
discard_tops ::= discard_top+
return ::= return_expr
discard_tops RETURN_VALUE
return ::= popb_return
return ::= pop_return
return ::= pop_ex_return
except_stmt ::= pop_ex_return
pop_return ::= POP_TOP return_expr RETURN_VALUE
popb_return ::= return_expr POP_BLOCK RETURN_VALUE
pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
# 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
lastl_stmt ::= ifpoplaststmtl
ifpoplaststmtl ::= testexpr POP_TOP c_stmts_opt
ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms
# Keep indices the same in ifelsestmtl
cf_pt ::= COME_FROM POP_TOP
ifelsestmtl ::= testexpr c_stmts cf_pt else_suite
for38 ::= expr get_iter store for_block JUMP_BACK
for38 ::= expr get_for_iter store for_block JUMP_BACK
for38 ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK
for38 ::= expr get_for_iter store for_block
forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite
forelsestmt38 ::= expr get_for_iter store for_block JUMP_BACK _come_froms
else_suite
forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
returns_in_except ::= _stmts except_return_value
except_return_value ::= POP_BLOCK return
except_return_value ::= expr POP_BLOCK RETURN_VALUE
whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK
# FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of
# for / forelse is used. Add come froms and check of add up control-flow detection phase.
async_forelse_stmt38 ::= expr
GET_AITER
SETUP_FINALLY
GET_ANEXT
LOAD_CONST
YIELD_FROM
POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms
whilestmt38 ::= _come_froms testexpr returns POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts JUMP_BACK
whilestmt38 ::= _come_froms testexpr l_stmts come_froms
store for_block
COME_FROM_FINALLY
END_ASYNC_FOR
else_suite
# while1elsestmt ::= l_stmts JUMP_BACK
whileTruestmt ::= _come_froms l_stmts JUMP_BACK POP_BLOCK
while1stmt ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE
whileTruestmt38 ::= _come_froms pass JUMP_BACK
# Seems to be used to discard values before a return in a "for" loop
discard_top ::= ROT_TWO POP_TOP
discard_tops ::= discard_top+
for_block ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK
return ::= return_expr
discard_tops RETURN_VALUE
except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false
POP_TOP POP_TOP POP_TOP
POP_EXCEPT
except_cond_as ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP STORE_FAST POP_TOP
return ::= popb_return
return ::= pop_return
return ::= pop_ex_return
except_stmt ::= pop_ex_return
pop_return ::= POP_TOP return_expr RETURN_VALUE
popb_return ::= return_expr POP_BLOCK RETURN_VALUE
pop_ex_return ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38 COME_FROM
else_suitel opt_come_from_except
try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38
# 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
lastl_stmt ::= ifpoplaststmtl
ifpoplaststmtl ::= testexpr POP_TOP c_stmts_opt
ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms
try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
except_handler38a
# Keep indices the same in ifelsestmtl
cf_pt ::= COME_FROM POP_TOP
ifelsestmtl ::= testexpr c_stmts cf_pt else_suite
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
try_except38r ::= SETUP_FINALLY return_except
except_handler38b
return_except ::= stmts POP_BLOCK return
for38 ::= expr get_iter store for_block JUMP_BACK
for38 ::= expr get_for_iter store for_block JUMP_BACK
for38 ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK
for38 ::= expr get_for_iter store for_block
forelsestmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suite
forelsestmt38 ::= expr get_for_iter store for_block JUMP_BACK _come_froms
else_suite
forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
returns_in_except ::= _stmts except_return_value
except_return_value ::= POP_BLOCK return
except_return_value ::= expr POP_BLOCK RETURN_VALUE
whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK
POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms
whilestmt38 ::= _come_froms testexpr returns POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts JUMP_BACK
whilestmt38 ::= _come_froms testexpr l_stmts come_froms
# while1elsestmt ::= l_stmts JUMP_BACK
whileTruestmt ::= _come_froms l_stmts JUMP_BACK POP_BLOCK
while1stmt ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK
whileTruestmt38 ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE
whileTruestmt38 ::= _come_froms pass JUMP_BACK
for_block ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK
except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false
POP_TOP POP_TOP POP_TOP
POP_EXCEPT
except_cond_as ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP STORE_FAST POP_TOP
try_elsestmtl38 ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38 COME_FROM
else_suitel opt_come_from_except
try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38
try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
except_handler38a
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
try_except38r ::= SETUP_FINALLY return_except
except_handler38b
return_except ::= stmts POP_BLOCK return
# In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
# is a final return in the "except" block.
# So we treat the "return" separate from the other statements
cond_except_stmt ::= except_cond1 except_stmts
cond_except_stmts_opt ::= cond_except_stmt*
# In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
# is a final return in the "except" block.
# So we treat the "return" separate from the other statements
cond_except_stmt ::= except_cond1 except_stmts
cond_except_stmts_opt ::= cond_except_stmt*
try_except38r2 ::= SETUP_FINALLY
suite_stmts_opt
POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
cond_except_stmts_opt
POP_EXCEPT return
END_FINALLY
COME_FROM
try_except38r2 ::= SETUP_FINALLY
suite_stmts_opt
POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
cond_except_stmts_opt
POP_EXCEPT return
END_FINALLY
COME_FROM
try_except38r3 ::= SETUP_FINALLY
suite_stmts_opt
POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY
cond_except_stmts_opt
POP_EXCEPT return
COME_FROM
END_FINALLY
COME_FROM
try_except38r3 ::= SETUP_FINALLY
suite_stmts_opt
POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY
cond_except_stmts_opt
POP_EXCEPT return
COME_FROM
END_FINALLY
COME_FROM
try_except38r4 ::= SETUP_FINALLY
returns_in_except
COME_FROM_FINALLY
except_cond1
return
COME_FROM
END_FINALLY
try_except38r4 ::= SETUP_FINALLY
returns_in_except
COME_FROM_FINALLY
except_cond1
return
COME_FROM
END_FINALLY
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler_as END_FINALLY COME_FROM
try_except_as ::= SETUP_FINALLY suite_stmts
except_handler_as END_FINALLY COME_FROM
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
try_except_as ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler_as END_FINALLY COME_FROM
try_except_as ::= SETUP_FINALLY suite_stmts
except_handler_as END_FINALLY COME_FROM
try_except_ret38 ::= SETUP_FINALLY returns except_ret38a
try_except_ret38a ::= SETUP_FINALLY returns except_handler38c
END_FINALLY come_from_opt
try_except_ret38 ::= SETUP_FINALLY returns except_ret38a
try_except_ret38a ::= SETUP_FINALLY returns except_handler38c
END_FINALLY come_from_opt
# Note: there is a suite_stmts_opt which seems
# to be bookkeeping which is not expressed in source code
except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
CALL_FINALLY RETURN_VALUE COME_FROM
COME_FROM_FINALLY
suite_stmts_opt END_FINALLY
except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
expr ROT_FOUR
POP_EXCEPT RETURN_VALUE END_FINALLY
# Note: there is a suite_stmts_opt which seems
# to be bookkeeping which is not expressed in source code
except_ret38 ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
CALL_FINALLY RETURN_VALUE COME_FROM
COME_FROM_FINALLY
suite_stmts_opt END_FINALLY
except_ret38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
expr ROT_FOUR
POP_EXCEPT RETURN_VALUE END_FINALLY
except_handler38 ::= _jump COME_FROM_FINALLY
except_stmts END_FINALLY opt_come_from_except
except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
POP_EXCEPT POP_TOP stmts END_FINALLY
except_handler38 ::= _jump COME_FROM_FINALLY
except_stmts END_FINALLY opt_come_from_except
except_handler38a ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
POP_EXCEPT POP_TOP stmts END_FINALLY
except_handler38c ::= COME_FROM_FINALLY except_cond1a except_stmts
POP_EXCEPT JUMP_FORWARD COME_FROM
except_handler_as ::= COME_FROM_FINALLY except_cond_as tryfinallystmt
POP_EXCEPT JUMP_FORWARD COME_FROM
except_handler38c ::= COME_FROM_FINALLY except_cond1a except_stmts
POP_EXCEPT JUMP_FORWARD COME_FROM
except_handler_as ::= COME_FROM_FINALLY except_cond_as tryfinallystmt
POP_EXCEPT JUMP_FORWARD COME_FROM
tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
END_FINALLY
tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
END_FINALLY
lc_setup_finally ::= LOAD_CONST SETUP_FINALLY
call_finally_pt ::= CALL_FINALLY POP_TOP
cf_cf_finally ::= come_from_opt COME_FROM_FINALLY
pop_finally_pt ::= POP_FINALLY POP_TOP
ss_end_finally ::= suite_stmts END_FINALLY
sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
lc_setup_finally ::= LOAD_CONST SETUP_FINALLY
call_finally_pt ::= CALL_FINALLY POP_TOP
cf_cf_finally ::= come_from_opt COME_FROM_FINALLY
pop_finally_pt ::= POP_FINALLY POP_TOP
ss_end_finally ::= suite_stmts END_FINALLY
sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
# FIXME: DRY rules below
tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally
ss_end_finally
tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally END_FINALLY
suite_stmts
tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally POP_FINALLY
ss_end_finally
tryfinally38rstmt ::= sf_bp_call_returns
COME_FROM_FINALLY POP_FINALLY
ss_end_finally
# FIXME: DRY rules below
tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally
ss_end_finally
tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally END_FINALLY
suite_stmts
tryfinally38rstmt ::= sf_pb_call_returns
cf_cf_finally POP_FINALLY
ss_end_finally
tryfinally38rstmt ::= sf_bp_call_returns
COME_FROM_FINALLY POP_FINALLY
ss_end_finally
tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
returns
cf_cf_finally pop_finally_pt
ss_end_finally POP_TOP
tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
COME_FROM COME_FROM_FINALLY
ss_end_finally
tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
returns
cf_cf_finally pop_finally_pt
ss_end_finally POP_TOP
tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
COME_FROM COME_FROM_FINALLY
ss_end_finally
tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY
POP_FINALLY suite_stmts_opt END_FINALLY
tryfinally38stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY
POP_FINALLY suite_stmts_opt END_FINALLY
tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY
POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
tryfinally38astmt ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
BEGIN_FINALLY COME_FROM_FINALLY
POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
"""
def p_38walrus(self, args):
@@ -361,13 +364,24 @@ class Python38Parser(Python37Parser):
"""
)
def customize_grammar_rules(self, tokens, customize):
super(Python37Parser, self).customize_grammar_rules(tokens, customize)
def customize_reduce_checks_full38(self, tokens, customize):
"""
Extra tests when a reduction is made in the full grammar.
Reductions here are extended from those used in the lambda grammar
"""
self.remove_rules_38()
self.check_reduce["pop_return"] = "tokens"
self.check_reduce["whileTruestmt38"] = "tokens"
self.check_reduce["whilestmt38"] = "tokens"
self.check_reduce["try_elsestmtl38"] = "AST"
self.reduce_check_table["pop_return"] = pop_return_check
def customize_grammar_rules(self, tokens, customize):
super(Python37Parser, self).customize_grammar_rules(tokens, customize)
self.customize_reduce_checks_full38(tokens, customize)
# For a rough break out on the first word. This may
# include instructions that don't need customization,
# but we'll do a finer check after the rough breakout.
@@ -422,11 +436,7 @@ class Python38Parser(Python37Parser):
# Determine if we have an iteration CALL_FUNCTION_1.
has_get_iter_call_function1 = False
for i, token in enumerate(tokens):
if (
token == "GET_ITER"
and i < n - 2
and tokens[i + 1] == "CALL_FUNCTION_1"
):
if token == "GET_ITER" and i < n - 2 and tokens[i + 1] == "CALL_FUNCTION_1":
has_get_iter_call_function1 = True
for i, token in enumerate(tokens):
@@ -524,26 +534,26 @@ class Python38Parser(Python37Parser):
continue
elif opname == "BUILD_STRING_2":
self.addRule(
"""
self.addRule(
"""
expr ::= formatted_value_debug
formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
""",
nop_func,
)
custom_ops_processed.add(opname)
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "BUILD_STRING_3":
self.addRule(
"""
self.addRule(
"""
expr ::= formatted_value_debug
formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3
formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3
""",
nop_func,
)
custom_ops_processed.add(opname)
nop_func,
)
custom_ops_processed.add(opname)
elif opname == "LOAD_CLOSURE":
self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
@@ -576,14 +586,10 @@ class Python38Parser(Python37Parser):
"""
self.addRule(rule, nop_func)
def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python38Parser,
self).reduce_is_invalid(rule, ast,
tokens, first, last)
invalid = super(Python38Parser, self).reduce_is_invalid(
rule, ast, tokens, first, last
)
self.remove_rules_38()
if invalid:
return invalid
@@ -611,7 +617,7 @@ if __name__ == "__main__":
p = Python38Parser()
p.remove_rules_38()
p.check_grammar()
from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
if PYTHON_VERSION_TRIPLE[:2] == (3, 8):
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
@@ -634,7 +640,9 @@ if __name__ == "__main__":
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
import sys
if len(sys.argv) > 1:
from spark_parser.spark import rule2str
for rule in sorted(p.rule2name.items()):
print(rule2str(rule[0]))

10
uncompyle6/parsers/reducecheck/pop_return.py Normal file
View File

@@ -0,0 +1,10 @@
# Copyright (c) 2020 Rocky Bernstein
def pop_return_check(
self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
) -> bool:
# If the first instruction of return_expr (the instruction after POP_TOP) is
# has a linestart, then the POP_TOP was probably part of the previous
# statement, such as a call() where the return value is discarded.
return tokens[first + 1].linestart

10
uncompyle6/scanner.py
View File

@@ -88,7 +88,7 @@ def long(num):
CONST_COLLECTIONS = ["CONST_LIST", "CONST_SET", "CONST_DICT", "CONST_MAP"]
class Code(object):
class Code:
"""
Class for representing code-objects.
@@ -291,7 +291,13 @@ class Scanner:
return False
return offset < self.get_target(offset)
def prev_offset(self, offset):
def ingest(self, co, classname=None, code_objects={}, show_asm=None):
"""
Code to tokenize disassembly. Subclasses must implement this.
"""
raise NotImplementedError("This method should have been implemented")
def prev_offset(self, offset: int) -> int:
return self.insts[self.offset2inst_index[offset] - 1].offset
def get_inst(self, offset):

6
uncompyle6/scanners/scanner3.py
View File

@@ -253,7 +253,7 @@ class Scanner3(Scanner):
has_arg=True,
has_extended_arg=False,
opc=self.opc,
optype=None,
optype="pseudo",
)
)
for j in range(collection_start, i):
@@ -280,7 +280,7 @@ class Scanner3(Scanner):
has_arg=t.has_arg,
has_extended_arg=False,
opc=t.opc,
optype=None,
optype="pseudo",
)
)
return new_tokens
@@ -327,7 +327,7 @@ class Scanner3(Scanner):
has_arg=True,
has_extended_arg=False,
opc=self.opc,
optype=None,
optype="pseudo",
)
)
for j in range(collection_start, i, 2):

4
uncompyle6/scanners/scanner37.py
View File

@@ -52,7 +52,7 @@ class Scanner37(Scanner37Base):
if collection_type == "CONST_DICT":
# constant dictionaries work via BUILD_CONST_KEY_MAP and
# handle the values() like sets and lists.
# However the keys() are an LOAD_CONST of the keys.
# However, the keys() are an LOAD_CONST of the keys.
# adjust offset to account for this
count += 1
@@ -89,7 +89,7 @@ class Scanner37(Scanner37Base):
has_arg=True,
has_extended_arg=False,
opc=self.opc,
optype=None,
optype="pseudo",
)
)
for j in range(collection_start, i):

7
uncompyle6/scanners/scanner37base.py
View File

@@ -263,10 +263,9 @@ class Scanner37Base(Scanner):
if (
next_inst.opname == "LOAD_GLOBAL"
and next_inst.argval == "AssertionError"
and inst.argval
and inst.argval is not None
):
raise_idx = self.offset2inst_index[self.prev_op[inst.argval]]
raise_inst = self.insts[raise_idx]
raise_inst = self.get_inst(self.prev_op[inst.argval])
if raise_inst.opname.startswith("RAISE_VARARGS"):
self.load_asserts.add(next_inst.offset)
pass
@@ -283,7 +282,7 @@ class Scanner37Base(Scanner):
# some backward jumps, are turned into forward jumps to another
# "extended arg" backward jump to the same location.
if inst.opname == "JUMP_FORWARD":
jump_inst = self.insts[self.offset2inst_index[inst.argval]]
jump_inst = self.get_inst(inst.argval)
if jump_inst.has_extended_arg and jump_inst.opname.startswith("JUMP"):
# Create a combination of the jump-to instruction and
# this one. Keep the position information of this instruction,

38
uncompyle6/semantics/gencomp.py
View File

@@ -21,9 +21,9 @@ from xdis import co_flags_is_async, iscode
from uncompyle6.parser import get_python_parser
from uncompyle6.scanner import Code
from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.consts import PRECEDENCE
from uncompyle6.semantics.helper import is_lambda_mode
from uncompyle6.scanners.tok import Token
class ComprehensionMixin(object):
@@ -179,7 +179,10 @@ class ComprehensionMixin(object):
tree = tree[1]
pass
if tree in ("genexpr_func", "genexpr_func_async",):
if tree in (
"genexpr_func",
"genexpr_func_async",
):
for i in range(3, 5):
if tree[i] == "comp_iter":
iter_index = i
@@ -340,8 +343,19 @@ class ComprehensionMixin(object):
assert store == "store"
n = set_iter_async[2]
elif node == "list_comp" and tree[0] == "expr":
tree = tree[0][0]
n = tree[iter_index]
list_iter = None
for list_iter_try in tree:
if list_iter_try == "list_iter":
list_iter = list_iter_try
break
if not list_iter_try:
tree = tree[0][0]
n = tree[iter_index]
else:
n = list_iter
pass
pass
pass
else:
n = tree[iter_index]
@@ -415,6 +429,9 @@ class ComprehensionMixin(object):
n = n[0]
if n in ("list_for", "comp_for"):
if n == "list_for" and not comp_for and n[0] == "expr":
comp_for = n[0]
n_index = 3
if (
(n[2] == "store")
@@ -504,11 +521,21 @@ class ComprehensionMixin(object):
if comp_for:
self.preorder(comp_for)
else:
try:
node[in_node_index]
except:
from trepan.api import debug
debug()
self.preorder(node[in_node_index])
# Here is where we handle nested list iterations.
if tree == "list_comp" and self.version != (3, 0):
list_iter = tree[1]
list_iter = None
for list_iter_try in tree:
if list_iter_try == "list_iter":
list_iter = list_iter_try
break
assert list_iter == "list_iter"
if list_iter[0] == "list_for":
self.preorder(list_iter[0][3])
@@ -648,7 +675,6 @@ class ComprehensionMixin(object):
# Find the list comprehension body. It is the inner-most
# node that is not list_.. .
while n == "list_iter":
# recurse one step
n = n[0]

1
uncompyle6/semantics/make_function2.py
View File

@@ -17,6 +17,7 @@
All the crazy things we have to do to handle Python functions in Python before 3.0.
The saga of changes continues in 3.0 and above and in other files.
"""
from xdis import code_has_star_arg, code_has_star_star_arg, iscode
from uncompyle6.scanner import Code

2
uncompyle6/semantics/n_actions.py
View File

@@ -1035,7 +1035,7 @@ class NonterminalActions:
self.prec = p
self.prune() # stop recursing
def n_listcomp(self, node):
def n_list_comp(self, node):
self.write("[")
if node[0].kind == "load_closure":
assert self.version >= (3, 0)

2
uncompyle6/semantics/pysource.py
View File

@@ -1275,7 +1275,7 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
transform_tree = self.treeTransform.transform(ast, code)
self.maybe_show_tree(ast, phase="before")
self.maybe_show_tree(transform_tree, phase="after")
del ast # Save memory
return transform_tree

16
uncompyle6/semantics/transform.py
View File

@@ -488,19 +488,9 @@ class TreeTransform(GenericASTTraversal, object):
self.ast[i] = self.ast[i][0]
if is_docstring(self.ast[i], self.version, code.co_consts):
load_const = self.ast[i].first_child()
docstring_ast = SyntaxTree(
"docstring",
[
Token(
"LOAD_STR",
has_arg=True,
offset=0,
attr=load_const.attr,
pattr=load_const.pattr,
)
],
)
load_const = copy(self.ast[i].first_child())
store_name = copy(self.ast[i].last_child())
docstring_ast = SyntaxTree("docstring", [load_const, store_name])
docstring_ast.transformed_by = "transform"
del self.ast[i]
self.ast.insert(0, docstring_ast)