Merge branch 'master' into python-3.3-to-3.5

This commit is contained in:
rocky
2022-06-08 12:53:50 -04:00
13 changed files with 347 additions and 165 deletions

View File

@@ -11,6 +11,17 @@ Bugs are not for asking questions about a problem you
are trying to solve that involve the use of uncompyle6 along the way,
although I may be more tolerent of this if you sponsor the project.
Bugs are also not for general or novice kind help on how to install
this Python program in your environment in the way you would like to
have it set up, or how to interpret a Python traceback e.g. that winds
up saying Python X.Y.Z is not supported.
For these kinds of things, you will save yourself time by asking
instead on forums like StackOverflow that are geared to helping people
for such general or novice kinds questions and tasks. And unless you
are a sponsor of the project, if your question seems to be of this
category, the issue may just be closed.
Also, the unless you are a sponsor of the project, it may take a
while, maybe a week or so, before the bug report is noticed, let alone
acted upon.
@@ -54,7 +65,7 @@ Bug reports that violate the above may be discarded.
## Description
<!-- Please add a clear and concise description of the bug. -->
<!-- Please add a clear and concise description of the bug. Try to narrow the problem down to the smallest that exhibits the bug.-->
## How to Reproduce

View File

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
echo "This script should be *sourced* rather than run directly through bash"
exit 1
fi
export PYVERSIONS='3.6.15 3.7.13 pypy3.6-7.3.0 pyston-2.3.2 3.8.13 3.9.12 3.10.4'
export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.13 pypy3.8-7.3.9 pyston-2.3.3 3.8.13'

View File

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
echo "This script should be *sourced* rather than run directly through bash"
exit 1
fi
export PYVERSIONS='3.7.11 3.8.12 3.9.7 3.10.0'
export PYVERSIONS='3.7.13 pyston-2.3.3 3.8.13'

View File

@@ -42,7 +42,7 @@ SKIP_TESTS=(
[test_decimal.py]=1 #
[test_dis.py]=1 # We change line numbers - duh!
[test_generators.py]=1 # Investigate
[test_grammar.py]=1 # Too many stmts. Handle large stmts
# [test_grammar.py]=1 # fails on its own - no module tests.test_support
[test_grp.py]=1 # Long test - might work Control flow?
[test_pep247.py]=1 # Long test - might work? Control flow?
[test_socketserver.py]=1 # -- test takes too long to run: 40 seconds

View File

@@ -44,7 +44,7 @@ SKIP_TESTS=(
[test_dis.py]=1 # We change line numbers - duh!
[test_file.py]=1 # test assertion failures
[test_generators.py]=1 # Investigate
[test_grammar.py]=1 # Too many stmts. Handle large stmts
# [test_grammar.py]=1 # fails on its own - no module tests.test_support
[test_grp.py]=1 # Long test - might work Control flow?
[test_macfs.py]=1 # it fails on its own
[test_macostools.py]=1 # it fails on its own

View File

@@ -9,7 +9,6 @@ SKIP_TESTS=(
[test_doctest.py]=1 # Fails on its own
[test_exceptions.py]=1
[test_format.py]=1 # Control flow "and" vs nested "if"
[test_grammar.py]=1 # Too many stmts. Handle large stmts
[test_grp.py]=1 # test takes to long, works interactively though
[test_io.py]=1 # Test takes too long to run
[test_ioctl.py]=1 # Test takes too long to run

View File

@@ -14,22 +14,19 @@ SKIP_TESTS=(
# complicated control flow and "and/or" expressions
[test_pickle.py]=1
[test_builtin.py]=1 # FIXME works on decompyle6
[test_context.py]=1 # FIXME works on decompyle6
[test_doctest2.py]=1 # FIXME works on decompyle6
[test_format.py]=1 # FIXME works on decompyle6
[test_marshal.py]=1 # FIXME works on decompyle6
[test_normalization.py]=1 # FIXME works on decompyle6
[test_os.py]=1 # FIXME works on decompyle6
[test_pow.py]=1 # FIXME works on decompyle6
[test_slice.py]=1 # FIXME works on decompyle6
[test_sort.py]=1 # FIXME works on decompyle6
[test_statistics.py]=1 # FIXME works on decompyle6
[test_timeit.py]=1 # FIXME works on decompyle6
[test_urllib2_localnet.py]=1 # FIXME works on decompyle6
[test_urllib2.py]=1 # FIXME: works on uncompyle6
[test_generators.py]=1 # FIXME: works on uncompyle6 - lambda parsing probably
[test_grammar.py]=1 # FIXME: works on uncompyle6 - lambda parsing probably
[test_generators.py]=1 # File "test_generators.py", line 44, in test_raise_and_yield_from self.assertEqual(exc.value, 'PASSED')
[test_grammar.py]=1 # FIXME: invalid syntax: l4 = lambda x=lambda y=lambda z=1: z: y(): x()
[test___all__.py]=1 # it fails on its own
[test_argparse.py]=1 #- it fails on its own
@@ -130,7 +127,6 @@ SKIP_TESTS=(
[test_traceback.py]=1 # Probably uses comment for testing
[test_tracemalloc.py]=1 # test assert failres
[test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds
[test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds
[test_typing.py]=1 # parse error
[test_types.py]=1 # parse error

View File

@@ -312,11 +312,6 @@ class Python26Parser(Python2Parser):
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP
jmp_false_then compare_chained2 _come_froms
return_lambda ::= RETURN_VALUE
return_lambda ::= RETURN_END_IF
return_lambda ::= RETURN_END_IF_LAMBDA
return_lambda ::= RETURN_VALUE_LAMBDA
compare_chained2 ::= expr COMPARE_OP return_expr_lambda
compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
compare_chained2 ::= expr COMPARE_OP RETURN_END_IF COME_FROM

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2017-2020 Rocky Bernstein
# Copyright (c) 2017-2020, 2022 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -22,34 +22,31 @@ from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parsers.parse37 import Python37Parser
class Python38Parser(Python37Parser):
def p_38walrus(self, args):
"""
# named_expr is also known as the "walrus op" :=
expr ::= named_expr
named_expr ::= expr DUP_TOP store
"""
def p_38misc(self, args):
def p_38_stmt(self, args):
"""
stmt ::= async_for_stmt38
stmt ::= async_forelse_stmt38
stmt ::= call_stmt
stmt ::= continue
stmt ::= for38
stmt ::= forelsestmt38
stmt ::= forelselaststmt38
stmt ::= forelselaststmtl38
stmt ::= tryfinally38stmt
stmt ::= forelsestmt38
stmt ::= try_elsestmtl38
stmt ::= try_except38
stmt ::= try_except38r
stmt ::= try_except38r2
stmt ::= try_except38r3
stmt ::= try_except38r4
stmt ::= try_except_as
stmt ::= try_except_ret38
stmt ::= tryfinally38astmt
stmt ::= tryfinally38rstmt
stmt ::= tryfinally38rstmt2
stmt ::= tryfinally38rstmt3
stmt ::= tryfinally38astmt
stmt ::= try_elsestmtl38
stmt ::= try_except_ret38
stmt ::= try_except38
stmt ::= try_except_as
stmt ::= whilestmt38
stmt ::= tryfinally38stmt
stmt ::= whileTruestmt38
stmt ::= call_stmt
stmt ::= continue
stmt ::= whilestmt38
call_stmt ::= call
break ::= POP_BLOCK BREAK_LOOP
@@ -129,6 +126,10 @@ class Python38Parser(Python37Parser):
forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec
forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
returns_in_except ::= _stmts except_return_value
except_return_value ::= POP_BLOCK return
except_return_value ::= expr POP_BLOCK RETURN_VALUE
whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK
whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms
@@ -155,9 +156,53 @@ class Python38Parser(Python37Parser):
else_suitel opt_come_from_except
try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
except_handler38
try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
except_handler38a
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
try_except38r ::= SETUP_FINALLY return_except
except_handler38b
return_except ::= stmts POP_BLOCK return
# In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
# is a final return in the "except" block.
# So we treat the "return" separate from the other statements
cond_except_stmt ::= except_cond1 except_stmts
cond_except_stmts_opt ::= cond_except_stmt*
try_except38r2 ::= SETUP_FINALLY
suite_stmts_opt
POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
cond_except_stmts_opt
POP_EXCEPT return
END_FINALLY
COME_FROM
try_except38r3 ::= SETUP_FINALLY
suite_stmts_opt
POP_BLOCK JUMP_FORWARD
COME_FROM_FINALLY
cond_except_stmts_opt
POP_EXCEPT return
COME_FROM
END_FINALLY
COME_FROM
try_except38r4 ::= SETUP_FINALLY
returns_in_except
COME_FROM_FINALLY
except_cond1
return
COME_FROM
END_FINALLY
# suite_stmts has a return
try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts
except_handler38b
@@ -234,6 +279,13 @@ class Python38Parser(Python37Parser):
POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
"""
def p_38walrus(self, args):
"""
# named_expr is also known as the "walrus op" :=
expr ::= named_expr
named_expr ::= expr DUP_TOP store
"""
def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
super(Python38Parser, self).__init__(debug_parser)
self.customized = {}

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2021 by Rocky Bernstein
# Copyright (c) 2021-2022 by Rocky Bernstein
"""
Python PyPy 3.8 decompiler scanner.
@@ -6,7 +6,7 @@ Does some additional massaging of xdis-disassembled instructions to
make things easier for decompilation.
"""
import decompyle3.scanners.scanner38 as scan
import uncompyle6.scanners.scanner38 as scan
# bytecode verification, verify(), uses JUMP_OPS from here
from xdis.opcodes import opcode_38pypy as opc

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2019-2020 by Rocky Bernstein
# Copyright (c) 2019-2020, 2022 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -25,133 +25,265 @@ def customize_for_version38(self, version):
# FIXME: pytest doesn't add proper keys in testing. Reinstate after we have fixed pytest.
# for lhs in 'for forelsestmt forelselaststmt '
# 'forelselaststmtl tryfinally38'.split():
# 'forelselaststmtc tryfinally38'.split():
# del TABLE_DIRECT[lhs]
TABLE_DIRECT.update({
TABLE_DIRECT.update(
{
"async_for_stmt38": (
"%|async for %c in %c:\n%+%c%-%-\n\n",
(2, "store"), (0, "expr"), (3, "for_block") ),
'async_forelse_stmt38': (
'%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
(7, 'store'), (0, 'expr'), (8, 'for_block'), (-1, 'else_suite') ),
(2, "store"),
(0, "expr"),
(3, "for_block"),
),
"async_forelse_stmt38": (
"%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(7, 'store'),
(0, 'expr'),
(8, 'for_block'),
(-1, 'else_suite')
),
"async_with_stmt38": (
"%|async with %c:\n%+%|%c%-",
(0, "expr"), 7),
"%|async with %c:\n%+%c%-\n",
(0, "expr"),
(7, ("l_stmts_opt", "l_stmts", "pass")),
),
"async_with_as_stmt38": (
"%|async with %c as %c:\n%+%|%c%-",
(0, "expr"), (6, "store"),
(7, "suite_stmts")
(0, "expr"),
(6, "store"),
(7, "suite_stmts"),
),
"c_forelsestmt38": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(2, "store"),
(0, "expr"),
(3, "for_block"),
-1,
),
"c_tryfinallystmt38": (
"%|try:\n%+%c%-%|finally:\n%+%c%-\n\n",
(1, "c_suite_stmts_opt"),
(-2, "c_suite_stmts_opt"),
),
"except_cond1a": ("%|except %c:\n", (1, "expr"),),
"except_cond_as": (
"%|except %c as %c:\n",
(1, "expr"),
(-2, "STORE_FAST"),
),
'except_handler38': (
'%c', (2, 'except_stmts') ),
'except_handler38a': (
'%c', (-2, 'stmts') ),
"except_handler38": ("%c", (2, "except_stmts")),
"except_handler38a": ("%c", (-2, "stmts")),
"except_handler38c": (
"%c%+%c%-",
(1, "except_cond1a"),
(2, "except_stmts"),
),
"except_handler_as": (
"%c%+\n%+%c%-",
(1, "except_cond_as"),
(2, "tryfinallystmt"),
),
'except_ret38a': (
'return %c', (4, 'expr') ),
"except_ret38a": ("return %c", (4, "expr")),
# Note: there is a suite_stmts_opt which seems
# to be bookkeeping which is not expressed in source code
'except_ret38': ( '%|return %c\n', (1, 'expr') ),
'for38': (
'%|for %c in %c:\n%+%c%-\n\n',
(2, 'store'),
(0, 'expr'),
(3, 'for_block') ),
"except_ret38": ("%|return %c\n", (1, "expr")),
"for38": (
"%|for %c in %c:\n%+%c%-\n\n",
(2, "store"),
(0, "expr"),
(3, "for_block"),
),
"forelsestmt38": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(2, "store"),
(0, "expr"),
(3, "for_block"), -1 ),
'forelselaststmt38': (
'%|for %c in %c:\n%+%c%-%|else:\n%+%c%-',
(2, 'store'),
(0, 'expr'),
(3, 'for_block'), -2 ),
'forelselaststmtl38': (
'%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n',
(2, 'store'),
(0, 'expr'),
(3, 'for_block'), -2 ),
'ifpoplaststmtl': ( '%|if %c:\n%+%c%-',
(0, "testexpr"),
(2, "c_stmts" ) ),
'ifstmtl': ( '%|if %c:\n%+%c%-',
(0, "testexpr"),
(1, "_ifstmts_jumpl") ),
'whilestmt38': ( '%|while %c:\n%+%c%-\n\n',
(1, 'testexpr'),
2 ), # "l_stmts" or "pass"
'whileTruestmt38': ( '%|while True:\n%+%c%-\n\n',
1 ), # "l_stmts" or "pass"
'try_elsestmtl38': (
'%|try:\n%+%c%-%c%|else:\n%+%c%-',
(1, 'suite_stmts_opt'),
(3, 'except_handler38'),
(5, 'else_suitel') ),
'try_except38': (
'%|try:\n%+%c\n%-%|except:\n%|%-%c\n\n',
(-2, 'suite_stmts_opt'), (-1, 'except_handler38a') ),
(3, "for_block"),
-1,
),
"forelselaststmt38": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-",
(2, "store"),
(0, "expr"),
(3, "for_block"),
-2,
),
"forelselaststmtc38": (
"%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n",
(2, "store"),
(0, "expr"),
(3, "for_block"),
-2,
),
"ifpoplaststmtc": ("%|if %c:\n%+%c%-", (0, "testexpr"), (2, "l_stmts")),
"pop_return": ("%|return %c\n", (1, "return_expr")),
"popb_return": ("%|return %c\n", (0, "return_expr")),
"pop_ex_return": ("%|return %c\n", (0, "return_expr")),
"set_for": (" for %c in %c", (2, "store"), (0, "expr_or_arg"),),
"whilestmt38": (
"%|while %c:\n%+%c%-\n\n",
(1, "testexpr"),
(2, ("l_stmts", "pass")),
),
"whileTruestmt38": ("%|while True:\n%+%c%-\n\n", (1, "l_stmts", "pass"),),
"try_elsestmtl38": (
"%|try:\n%+%c%-%c%|else:\n%+%c%-",
(1, "suite_stmts_opt"),
(3, "except_handler38"),
(5, "else_suitel"),
),
"try_except38": (
"%|try:\n%+%c\n%-%|except:\n%+%c%-\n\n",
(2, ("suite_stmts_opt", "suite_stmts")),
(3, ("except_handler38a", "except_handler38b", "except_handler38c")),
),
"try_except38r": (
"%|try:\n%+%c\n%-%|except:\n%+%c%-\n\n",
(1, "return_except"),
(2, "except_handler38b"),
),
"try_except38r2": (
"%|try:\n%+%c\n%-%|except:\n%+%c%c%-\n\n",
(1, "suite_stmts_opt"),
(8, "cond_except_stmts_opt"),
(10, "return"),
),
"try_except38r4": (
"%|try:\n%+%c\n%-%|except:\n%+%c%c%-\n\n",
(1, "returns_in_except"),
(3, "except_cond1"),
(4, "return"),
),
"try_except_as": (
"%|try:\n%+%c%-\n%|%-%c\n\n",
(-4, "suite_stmts"), # Go from the end because of POP_BLOCK variation
(
-4,
("suite_stmts", "_stmts"),
), # Go from the end because of POP_BLOCK variation
(-3, "except_handler_as"),
),
"try_except_ret38": (
"%|try:\n%+%c%-\n%|except:\n%+%|%c%-\n\n",
(1, "returns"),
(2, "except_ret38a"),
),
'tryfinally38rstmt': (
'%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
"try_except_ret38a": (
"%|try:\n%+%c%-%c\n\n",
(1, "returns"),
(2, "except_handler38c"),
),
"tryfinally38rstmt": (
"%|try:\n%+%c%-%|finally:\n%+%c%-\n\n",
(0, "sf_pb_call_returns"),
(-1, ("ss_end_finally", "suite_stmts")),
(-1, ("ss_end_finally", "suite_stmts", "_stmts")),
),
"tryfinally38rstmt2": (
"%|try:\n%+%c%-%|finally:\n%+%c%-\n\n",
(4, "returns"),
-2, "ss_end_finally"
-2,
"ss_end_finally",
),
"tryfinally38rstmt3": (
"%|try:\n%+%|return %c%-\n%|finally:\n%+%c%-\n\n",
(1, "expr"),
(-1, "ss_end_finally")
(-1, "ss_end_finally"),
),
'tryfinally38stmt': (
'%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
"tryfinally38rstmt4": (
"%|try:\n%+%c%-\n%|finally:\n%+%c%-\n\n",
(1, "suite_stmts_opt"),
(6, "suite_stmts_opt") ),
'tryfinally38astmt': (
'%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
(5, "suite_stmts_return"),
),
"tryfinally38stmt": (
"%|try:\n%+%c%-%|finally:\n%+%c%-\n\n",
(1, "suite_stmts_opt"),
(6, "suite_stmts_opt"),
),
"tryfinally38astmt": (
"%|try:\n%+%c%-%|finally:\n%+%c%-\n\n",
(2, "suite_stmts_opt"),
(8, "suite_stmts_opt") ),
(8, "suite_stmts_opt"),
),
"named_expr": ( # AKA "walrus operator"
"%c := %p", (2, "store"), (0, "expr", PRECEDENCE["named_expr"]-1)
"%c := %p",
(2, "store"),
(0, "expr", PRECEDENCE["named_expr"] - 1),
),
}
)
})
def except_return_value(node):
if node[0] == "POP_BLOCK":
self.default(node[1])
else:
self.template_engine(("%|return %c\n", (0, "expr")), node)
self.prune()
self.n_except_return_value = except_return_value
# FIXME: now that we've split out cond_except_stmt,
# we should be able to get this working as a pure transformation rule,
# so no procedure is needed here.
def try_except38r3(node):
self.template_engine(("%|try:\n%+%c\n%-", (1, "suite_stmts_opt")), node)
cond_except_stmts_opt = node[5]
assert cond_except_stmts_opt == "cond_except_stmts_opt"
for child in cond_except_stmts_opt:
if child == "cond_except_stmt":
if child[0] == "except_cond1":
self.template_engine(
("%c\n", (0, "except_cond1"), (1, "expr")), child
)
self.template_engine(("%+%c%-\n", (1, "except_stmts")), child)
pass
pass
self.template_engine(("%+%c%-\n", (7, "return")), node)
self.prune()
self.n_try_except38r3 = try_except38r3
def n_list_afor(node):
if len(node) == 2:
# list_afor ::= get_iter list_afor
self.comprehension_walk_newer(node, 0)
else:
list_iter_index = 2 if node[2] == "list_iter" else 3
self.template_engine(
(
" async for %[1]{%c} in %c%[1]{%c}",
(1, "store"),
(0, "get_aiter"),
(list_iter_index, "list_iter"),
),
node,
)
self.prune()
self.n_list_afor = n_list_afor
def n_set_afor(node):
if len(node) == 2:
self.template_engine(
(" async for %[1]{%c} in %c", (1, "store"), (0, "get_aiter")), node
)
else:
self.template_engine(
" async for %[1]{%c} in %c%c",
(1, "store"),
(0, "get_aiter"),
(2, "set_iter"),
)
self.prune()
self.n_set_afor = n_set_afor
def n_suite_stmts_return(node):
if len(node) > 1:
assert len(node) == 2
self.template_engine(
("%c\n%|return %c", (0, ("_stmts", "suite_stmts")), (1, "expr")), node
)
else:
self.template_engine(("%|return %c", (0, "expr")), node)
self.prune()
self.n_suite_stmts_return = n_suite_stmts_return

View File

@@ -1167,11 +1167,8 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin):
# f'{(lambda x:x)("8")!r}'
# Adding a "\n" after "lambda x: x" will give an error message:
# SyntaxError: f-string expression part cannot include a backslash
# So avoid that.
printfn = (
self.write if self.in_format_string and is_lambda else self.println
)
printfn(self.text)
# So avoid \n after writing text
self.write(self.text)
self.name = old_name
self.return_none = rn