diff --git a/.circleci/config.yml b/.circleci/config.yml index c2dfe464..a1466d27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,11 +10,13 @@ jobs: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results COMPILE: --compile + PYENV_ROOT: "$HOME/.pyenv" + PATH: "$PYENV_ROOT/bin:$PYENV_ROOT/shims:/usr/bin:/usr/local/bin:/sbin:/usr/sbin" + # To see the list of pre-built images that CircleCI provides for most common languages see # https://circleci.com/docs/2.0/circleci-images/ - machine: - python: - version: 2.7.14 + docker: + - image: circleci/jessie steps: # Machine Setup # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each @@ -24,7 +26,7 @@ jobs: # In many cases you can simplify this from what is generated here. # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - # This is based on your 1.0 configuration file or project settings + - run: curl https://pyenv.run | bash - run: working_directory: ~/rocky/python-uncompyle6 command: pyenv install 2.4.6 && pyenv local 2.4.6 && pyenv rehash && easy_install nose && pyenv rehash diff --git a/test/.gitignore b/test/.gitignore index cf825a74..3b34597f 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,2 +1,3 @@ /.coverage +/.python-version /nohup.out diff --git a/test/bytecode_2.7/05_abc_class.pyc b/test/bytecode_2.7/05_abc_class.pyc index 936ba1b0..7e18741e 100644 Binary files a/test/bytecode_2.7/05_abc_class.pyc and b/test/bytecode_2.7/05_abc_class.pyc differ diff --git a/test/bytecode_3.0_run/05_block_fallback.pyc b/test/bytecode_3.0_run/05_block_fallback.pyc new file mode 100644 index 00000000..b2f9418a Binary files /dev/null and b/test/bytecode_3.0_run/05_block_fallback.pyc differ diff --git a/test/bytecode_3.5/02_async_for.pyc b/test/bytecode_3.5/02_async_for.pyc index b2908f7d..fee1c418 100644 Binary files a/test/bytecode_3.5/02_async_for.pyc and b/test/bytecode_3.5/02_async_for.pyc differ diff --git a/test/bytecode_3.5_run/05_block_fallback.pyc b/test/bytecode_3.5_run/05_block_fallback.pyc new file mode 100644 index 00000000..186cc2c0 Binary files /dev/null and b/test/bytecode_3.5_run/05_block_fallback.pyc differ diff --git a/test/bytecode_3.6/02_async_for.pyc b/test/bytecode_3.6/02_async_for.pyc index 857c1417..9434edd3 100644 Binary files a/test/bytecode_3.6/02_async_for.pyc and b/test/bytecode_3.6/02_async_for.pyc differ diff --git a/test/bytecode_3.6/04_async.pyc b/test/bytecode_3.6/04_async.pyc new file mode 100644 index 00000000..ca37ce39 Binary files /dev/null and b/test/bytecode_3.6/04_async.pyc differ diff --git a/test/bytecode_3.7/02_async_for.pyc b/test/bytecode_3.7/02_async_for.pyc index 019a14cc..25866060 100644 Binary files a/test/bytecode_3.7/02_async_for.pyc and b/test/bytecode_3.7/02_async_for.pyc differ diff --git a/test/bytecode_3.7/04_async.pyc b/test/bytecode_3.7/04_async.pyc index 0b2e13b7..67f813ed 100644 Binary files a/test/bytecode_3.7/04_async.pyc and b/test/bytecode_3.7/04_async.pyc differ diff --git a/test/bytecode_3.7_run/05_block_fallback.pyc b/test/bytecode_3.7_run/05_block_fallback.pyc new file mode 100644 index 00000000..fd4692f7 Binary files /dev/null and b/test/bytecode_3.7_run/05_block_fallback.pyc differ diff --git a/test/bytecode_3.8/04_async.pyc b/test/bytecode_3.8/04_async.pyc new file mode 100644 index 00000000..2e6857c1 Binary files /dev/null and b/test/bytecode_3.8/04_async.pyc differ diff --git a/test/simple_source/branching/02_ifelse_lambda.py b/test/simple_source/branching/02_ifelse_lambda.py index 6b68ee52..9de82c93 100644 --- a/test/simple_source/branching/02_ifelse_lambda.py +++ b/test/simple_source/branching/02_ifelse_lambda.py @@ -22,7 +22,7 @@ assert i[0]('a') == True assert i[0]('A') == False # Issue #170. Bug is needing an "conditional_not_lambda" grammar rule -# in addition the the "if_expr_lambda" rule +# in addition the the "if_exp_lambda" rule j = lambda a: False if not a else True assert j(True) == True assert j(False) == False diff --git a/test/simple_source/branching/10_if_else_ternary.py b/test/simple_source/branching/10_if_else_ternary.py index 7f1b02ea..b1753d9f 100644 --- a/test/simple_source/branching/10_if_else_ternary.py +++ b/test/simple_source/branching/10_if_else_ternary.py @@ -1,8 +1,8 @@ # Tests: # ret_expr_or_cond ::= ret_expr -# ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond -# ret_expr_or_cond ::= ret_cond +# if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond +# ret_expr_or_cond ::= if_exp_ret # ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM # See https://github.com/rocky/python-uncompyle6/issues/5 diff --git a/test/simple_source/bug26/03_weird26.py b/test/simple_source/bug26/03_weird26.py index 34e176b0..9b583eed 100644 --- a/test/simple_source/bug26/03_weird26.py +++ b/test/simple_source/bug26/03_weird26.py @@ -8,7 +8,7 @@ list(x for x in range(10) if x % 2 if x % 3) # expresion which evaluates True unconditionally, # but leave dead code or junk around that we have to match on. -# Tests "if_expr_true" rule +# Tests "if_exp_true" rule 5 if 1 else 2 0 or max(5, 3) if 0 else 3 diff --git a/test/simple_source/bug27+/03_not_dead_code.py b/test/simple_source/bug27+/03_not_dead_code.py index f4698761..61a2309c 100644 --- a/test/simple_source/bug27+/03_not_dead_code.py +++ b/test/simple_source/bug27+/03_not_dead_code.py @@ -1,6 +1,6 @@ # Bug found in 2.7 test_itertools.py -# Bug was erroneously using reduction to if_expr_true -# A proper fix would be to use if_expr_true only when we +# Bug was erroneously using reduction to if_exp_true +# A proper fix would be to use if_exp_true only when we # can determine there is or was dead code. from itertools import izip_longest for args in [['abc', range(6)]]: diff --git a/test/simple_source/bug30/05_block_fallback.py b/test/simple_source/bug30/05_block_fallback.py new file mode 100644 index 00000000..ed1c979c --- /dev/null +++ b/test/simple_source/bug30/05_block_fallback.py @@ -0,0 +1,32 @@ +# Adapted from 3.7.6 test_contains +# The bug was in reconstructing something equivalent to +# "while False: yelid None" which is *needed* in __iter__(). +# Sheeh! + +# RUNNABLE! +def test_block_fallback(): + # blocking fallback with __contains__ = None + class ByContains(object): + def __contains__(self, other): + return False + c = ByContains() + class BlockContains(ByContains): + """Is not a container + + This class is a perfectly good iterable (as tested by + list(bc)), as well as inheriting from a perfectly good + container, but __contains__ = None prevents the usual + fallback to iteration in the container protocol. That + is, normally, 0 in bc would fall back to the equivalent + of any(x==0 for x in bc), but here it's blocked from + doing so. + """ + def __iter__(self): + while False: + yield None + __contains__ = None + bc = BlockContains() + assert not (0 in c) + assert not (0 in list(bc)) + +test_block_fallback() diff --git a/test/simple_source/bug35/02_async_for.py b/test/simple_source/bug35/02_async_for.py index 7b250d35..b8fbb1b3 100644 --- a/test/simple_source/bug35/02_async_for.py +++ b/test/simple_source/bug35/02_async_for.py @@ -1,3 +1,12 @@ async def a(b, c): async for b in c: pass + +# From 3.7 test_generators.py +# Bug was getting indentation correct for multiple async's +async def foo(X): + async for i in X: + pass + async for i in X: + pass + raise Done diff --git a/test/simple_source/bug37/04_async.py b/test/simple_source/bug37/04_async.py index eb46010b..d7947b19 100644 --- a/test/simple_source/bug37/04_async.py +++ b/test/simple_source/bug37/04_async.py @@ -10,3 +10,14 @@ async def test_enter(self): x = 1 y = 2 assert manager is context + +# From 3.7.6 test_coroutines.py +# Bug was different form of code for "async with" below +class CoroutineTest(): + def test_with_8(self): + CNT = 0 + async def foo(): + nonlocal CNT + async with CM(): + CNT += 1 + return diff --git a/test/simple_source/def/05_abc_class.py b/test/simple_source/def/05_abc_class.py index 0d277593..7e67d93a 100644 --- a/test/simple_source/def/05_abc_class.py +++ b/test/simple_source/def/05_abc_class.py @@ -12,3 +12,9 @@ class abstractclassmethod(classmethod): def __init__(self, callable): callable.__isabstractmethod__ = True super().__init__(callable) + +# From 2.7.17 test_abc.py +# Bug was handling OldstyleClass correctly without +# a "return locals() but with a "pass" +class OldstyleClass: + pass diff --git a/test/stdlib/2.4-exclude.sh b/test/stdlib/2.4-exclude.sh index 19c37eb3..77134222 100644 --- a/test/stdlib/2.4-exclude.sh +++ b/test/stdlib/2.4-exclude.sh @@ -52,3 +52,4 @@ SKIP_TESTS=( [test_trace.py]=1 # Long test - works [test_zipfile64.py]=1 # Runs ok but takes 204 seconds ) + # About 243 files, 0 in 19 minutes diff --git a/test/stdlib/2.5-exclude.sh b/test/stdlib/2.5-exclude.sh index 9d9af6a4..c2374ecd 100644 --- a/test/stdlib/2.5-exclude.sh +++ b/test/stdlib/2.5-exclude.sh @@ -79,6 +79,7 @@ SKIP_TESTS=( [test_zipfile64.py]=1 # Runs ok but takes 204 seconds [test_zlib]=1 # fails on its own ) +# About 265 tests in 14 minutes if (( batch )) ; then SKIP_TESTS[test_doctest.py]=1 # Fails on ppc64le diff --git a/test/stdlib/3.2-exclude.sh b/test/stdlib/3.2-exclude.sh index 49269171..821118d2 100644 --- a/test/stdlib/3.2-exclude.sh +++ b/test/stdlib/3.2-exclude.sh @@ -1,14 +1,5 @@ SKIP_TESTS=( - [test_decorators.py]=1 # FIXME: Works on c90ff51 - [test_optparse.py]=1 # FIXME: Works on c90ff51 - [test_os.py]=1 # FIXME: Works on c90ff51 - [test_pyclbr.py]=1 # FIXME: Works on c90ff51 - [test_strftime.py]=1 # FIXME: Works on c90ff51 - - [test_binop.py]=1 # FIXME: Works on c90ff51? - [test_cgi.py]=1 # FIXME: Works on c90ff51? [test_descr.py]=1 # FIXME: Works on c90ff51? - [test_doctest2.py]=1 # FIXME: Works on c90ff51? [test_cmd_line.py]=1 [test_collections.py]=1 diff --git a/test/stdlib/3.3-exclude.sh b/test/stdlib/3.3-exclude.sh index 3c436195..3ffc9191 100644 --- a/test/stdlib/3.3-exclude.sh +++ b/test/stdlib/3.3-exclude.sh @@ -1,16 +1,4 @@ SKIP_TESTS=( - [test_binop.py]=1 # FIXME: Works on c90ff51 - [test_cgi.py]=1 # FIXME: Works on c90ff51 - [test_decorators.py]=1 # FIXME: Works on c90ff51 - [test_pyclbr.py]=1 # FIXME: Works on c90ff51 - [test_optparse.py]=1 # FIXME: Works on c90ff51 - [test_os.py]=1 # FIXME: Works on c90ff51 - [test_pep352.py]=1 # FIXME: Works on c90ff51 - [test_pyclbr.py]=1 # FIXME: Works on c90ff51 - [test_shutil.py]=1 # FIXME: Works on c90ff51 - [test_strftime.py]=1 # FIXME: Works on c90ff51 - [test_symtable.py]=1 # FIXME: Works on c90ff51 - [test_atexit.py]=1 # The atexit test starting at 3.3 looks for specific comments in error lines [test_buffer.py]=1 # parse error diff --git a/test/stdlib/3.4-exclude.sh b/test/stdlib/3.4-exclude.sh index 1b713b3a..c616c441 100644 --- a/test/stdlib/3.4-exclude.sh +++ b/test/stdlib/3.4-exclude.sh @@ -1,13 +1,4 @@ SKIP_TESTS=( - [test_buffer.py]=1 # FIXME: Works on c90ff51 - [test_decorators.py]=1 # FIXME: Works on c90ff51 - [test_faulthandler.py]=1 # FIXME: too long to run 20 seconds. Works on c90ff51 - [test_decimal.py]=1 # FIXME: Works on c90ff51 - [test_optparse.py]=1 # FIXME: Works on c90ff51 - [test_os.py]=1 # FIXME: Works on c90ff51 - [test_shutil.py]=1 # FIXME: Works on c90ff51 - [test_strftime.py]=1 # FIXME: Works on c90ff51 - [test___all__.py]=1 # it fails on its own [test_atexit.py]=1 # The atexit test looks for specific comments in error lines [test_cmd_line.py]=1 # takes too long to run diff --git a/test/stdlib/3.5-exclude.sh b/test/stdlib/3.5-exclude.sh index 06e0b3c2..404e419c 100644 --- a/test/stdlib/3.5-exclude.sh +++ b/test/stdlib/3.5-exclude.sh @@ -1,20 +1,11 @@ SKIP_TESTS=( - [test_buffer.py]=1 # FIXME: Works on c90ff51 - [test_decorators.py]=1 # FIXME: Works on c90ff51 - [test_faulthandler.py]=1 # FIXME: too long to run 20 seconds. Works on c90ff51 - [test_ftplib.py]=1 # Works on c90ff51 - [test_marshal.py]=1 # FIXME: Works on c90ff51 - [test_optparse.py]=1 # FIXME: Works on c90ff51 - [test_os.py]=1 # FIXME: Works on c90ff51 + [test_buffer.py]=1 # FIXME: Works on c90ff51 + [test_decorators.py]=1 # FIXME: Works on c90ff51 [test_platform.py]=1 # FIXME: Works on c90ff51 - [test_poplib.py]=1 # FIXME: Works on c90ff51 [test_pyclbr.py]=1 # FIXME: Works on c90ff51 - [test_smtplib.py]=1 # FIXME: Works on c90ff51 - [test_strftime.py]=1 # FIXME: Works on c90ff51 - [test_shutil.py]=1 # FIXME: Works on c90ff51 - [test_sysconfig.py]=1 # FIXME: Works on c90ff51 [test_tempfile.py]=1 # FIXME: Works on c90ff51 [test_uu.py]=1 # FIXME: Works on c90ff51 + [test_ftplib.py]=1 # FIXME: Works on c90ff51 [test___all__.py]=1 # it fails on its own [test_aifc.py]=1 # @@ -33,7 +24,6 @@ SKIP_TESTS=( [test_collections.py]=1 [test_compile.py]=1 # Code introspects on co_consts in a non-decompilable way [test_concurrent_futures.py]=1 # Takes long to run - [test_coroutines.py]=1 # Syntax error Investigate [test_curses.py]=1 # [test_devpoll.py]=1 # it fails on its own @@ -50,6 +40,7 @@ SKIP_TESTS=( [test_exceptions.py]=1 # parse error [test_format.py]=1 + [test_ftplib.py]=1 # Test assertion failures [test_gdb.py]=1 # it fails on its own [test_glob.py]=1 # @@ -65,6 +56,7 @@ SKIP_TESTS=( [test_logging.py]=1 # [test_long.py]=1 # too long run time: 20 seconds + [test_marshal.py]=1 # test assertion errors [test_math.py]=1 # test assertion errors TypeError: a float is required [test_modulefinder.py]=1 # test assertion error [test_msilib.py]=1 # it fails on its own @@ -99,7 +91,7 @@ SKIP_TESTS=( [test_selectors.py]=1 # Takes too long 17 seconds [test_set.py]=1 # # test assert failure and doesn't terminate [test_signal.py]=1 # too long? - [test_smtpd.py]=1 # test failures + [test_smtplib.py]=1 # probably control flow [test_socket.py]=1 # long [test_socketserver.py]=1 [test_strtod.py]=1 # Test assert failure @@ -140,12 +132,13 @@ SKIP_TESTS=( if (( batch )) ; then SKIP_TESTS[test_asyncore.py]=1 # Ok, but takes more than 15 seconds to run SKIP_TESTS[test_bisect.py]=1 - SKIP_TESTS[test_buffer.py]=1 # too long SKIP_TESTS[test_compileall.py]=1 # Something weird on POWER SKIP_TESTS[test_codeccallbacks.py]=1 # Something differenet in locale? SKIP_TESTS[test_distutils.py]=1 SKIP_TESTS[test_exception_variations.py]=1 + SKIP_TESTS[test_poplib.py]=1 # May be a result of POWER installation + SKIP_TESTS[test_quopri.py]=1 SKIP_TESTS[test_ioctl.py]=1 # it fails on its own SKIP_TESTS[test_tarfile.py]=1 # too long to run on POWER 15 secs diff --git a/test/stdlib/3.6-exclude.sh b/test/stdlib/3.6-exclude.sh index 919f85a8..b85ab0ce 100644 --- a/test/stdlib/3.6-exclude.sh +++ b/test/stdlib/3.6-exclude.sh @@ -1,14 +1,13 @@ SKIP_TESTS=( + [test_ast.py]=1 # FIXME: Works on c90ff51 [test_binop.py]=1 # FIXME: Works on c90ff51 - [test_cmath.py]=1 # FIXME: Works on c90ff51 - [test_decorators.py]=1 # FIXME: Works on c90ff51 - [test_format.py]=1 # FIXME: works on c90ff51 - [test_locale.py]=1 # FIXME: Works on c90ff51 - [test_optparse.py]=1 # FIXME: Works on c90ff51 - [test_os.py]=1 # FIXME: Works on c90ff51 - [test_pyclbr.py]=1 # FIXME: Works on c90ff51 + [test_complex.py]=1 # FIXME: Works on c90ff51 + [test_format.py]=1 # FIXME: Works on c90ff51 + [test_ftplib.py]=1 # FIXME: Works on c90ff51 + [test_slice.py]=1 # FIXME: Works on c90ff51 [test_sort.py]=1 # FIXME: Works on c90ff51 - [test_strftime.py]=1 # FIXME: Works on c90ff51 + [test_timeit.py]=1 # FIXME: Works on c90ff51 + [test_os.py]=1 # FIXME: Works on c90ff51 [test___all__.py]=1 # it fails on its own [test_aifc.py]=1 # @@ -43,7 +42,7 @@ SKIP_TESTS=( [test_concurrent_futures.py]=1 # Takes long [test_contextlib.py]=1 # test assertion failure [test_contextlib_async.py]=1 # Investigate - [test_coroutines.py]=1 # Parse error + [test_coroutines.py]=1 # parse error [test_curses.py]=1 # Parse error [test_ctypes.py]=1 # it fails on its own @@ -182,6 +181,7 @@ SKIP_TESTS=( [test_urllib2.py]=1 # it fails on its own [test_urllibnet.py]=1 # it fails on its own [test_urllib.py]=1 # it fails on its own + [test_urlparse.py]=1 # test failure [test_venv.py]=1 # test takes too long to run: 13 seconds diff --git a/test/stdlib/3.7-exclude.sh b/test/stdlib/3.7-exclude.sh index 38d3c6da..2b1f0c19 100644 --- a/test/stdlib/3.7-exclude.sh +++ b/test/stdlib/3.7-exclude.sh @@ -1,58 +1,65 @@ SKIP_TESTS=( + [test_builtin.py]=1 # FIXME works on decompyle6 + [test_context.py]=1 # FIXME works on decompyle6 + [test_format.py]=1 # FIXME works on decompyle6 + [test_marshal.py]=1 # FIXME works on decompyle6 + [test_normalization.py]=1 # FIXME works on decompyle6 + [test_os.py]=1 # FIXME works on decompyle6 + [test_pow.py]=1 # FIXME works on decompyle6 + [test_slice.py]=1 # FIXME works on decompyle6 + [test_sort.py]=1 # FIXME works on decompyle6 + [test_statistics.py]=1 # FIXME works on decompyle6 + [test_string_literals.py]=1 # FIXME works on decompyle6 + [test_timeit.py]=1 # FIXME works on decompyle6 + [test_urllib2_localnet.py]=1 # FIXME works on decompyle6 + [test_urllib2.py]=1 # FIXME: works on uncompyle6 + [test_generators.py]=1 # Investigate improper lamdba with bogus "False" added + [test_grammar.py]=1 # investigate: like above: semantic rule missing probably + [test___all__.py]=1 # it fails on its own [test_argparse.py]=1 #- it fails on its own [test_asdl_parser.py]=1 # it fails on its own - [test_ast.py]=1 # Depends on comments in code [test_atexit.py]=1 # The atexit test looks for specific comments in error lines [test_baseexception.py]=1 # UnboundLocalError: local variable 'exc' referenced before assignment [test_bdb.py]=1 # - [test_buffer.py]=1 # test assertion errors - [test_builtin.py]=1 # parse error, but decompyle3 doesn't have this. (It has test assert failures though) + [test_buffer.py]=1 # parse error [test_clinic.py]=1 # it fails on its own [test_cmath.py]=1 # test assertion failure [test_cmd_line.py]=1 # Interactive? [test_cmd_line_script.py]=1 - [test_compare.py]=1 # Weird test assert faiure AssertionError: [1] == [1] [test_compileall.py]=1 # fails on its own [test_compile.py]=1 # Code introspects on co_consts in a non-decompilable way [test_concurrent_futures.py]=1 # too long - [test_context.py]=1 - [test_coroutines.py]=1 # Investigate: Parse error - async/yield stuff? + [test_coroutines.py]=1 # parse error [test_codecs.py]=1 # test assert failures; encoding/decoding stuff [test_ctypes.py]=1 # it fails on its own [test_curses.py]=1 # probably byte string not handled properly [test_dataclasses.py]=1 # FIXME: control flow probably: AssertionError: unknown result 'exception' [test_datetime.py]=1 # Takes too long - [test_dbm_gnu.py]=1 # Takes too long [test_dbm_ndbm.py]=1 # it fails on its own - [test_decimal.py]=1 # test assertion failures + [test_decimal.py]=1 # parse error [test_descr.py]=1 # test assertion failures [test_devpoll.py]=1 # it fails on its own - [test_dis.py]=1 # We change line numbers - duh! - [test_doctest2.py]=1 # FIXME: assert failure - works on decompyle3 + [test_dis.py]=1 # Investigate async out of place. Then We change line numbers - duh! + [test_doctest.py]=1 # test failures [test_docxmlrpc.py]=1 + [test_enum.py]=1 # probably bad control flow - [test_faulthandler.py]=1 # takes too long - [test_fcntl.py]=1 + [test_faulthandler.py]=1 # test takes too long before decompiling [test_fileinput.py]=1 # Test assertion failures - [test_format.py]=1 # Probably not handling bytestrings properly [test_frame.py]=1 # test assertion errors [test_ftplib.py]=1 # parse error [test_fstring.py]=1 # need to disambiguate leading fstrings from docstrings [test_functools.py]=1 # parse error [test_gdb.py]=1 # it fails on its own - [test_generators.py]=1 # Investigate improper lamdba with bogus "False" added [test_glob.py]=1 # TypeError: join() argument must be str or bytes, not 'tuple' - [test_grammar.py]=1 # investigate: index out of range in decompiler (template_engine?) [test_grp.py]=1 # Doesn't terminate (killed) - [test_hashlib.py]=1 # test assert failures - [test_imaplib-3.7.py]=1 # test assert failures - [test_idle.py]=1 # Probably installation specific + [test_imaplib.py]=1 # test run loops before decompiling? More than 15 seconds to run [test_io.py]=1 # test takes too long to run: 37 seconds - [test_imaplib.py]=1 # test assert failures + [test_imaplib.py]=1 # decompiled test loops - killing after 15 seconds [test_inspect.py]=1 # Investigate test failures involving lambda [test_kqueue.py]=1 # it fails on its own @@ -61,79 +68,62 @@ SKIP_TESTS=( [test_long.py]=1 # FIX: if boundaries wrong in Rat __init__ [test_logging.py]=1 # test takes too long to run: 20 seconds - [test_mailbox.py]=1 - [test_marshal.py]=1 + [test_mailbox.py]=1 # probably control flow [test_math.py]=1 # test assert failures - [test_modulefinder.py]=1 - [test_msilib.py]=1 + [test_msilib.py]=1 # it fails on its own [test_multiprocessing_fork.py]=1 # test takes too long to run: 62 seconds [test_multiprocessing_forkserver.py]=1 [test_multiprocessing_spawn.py]=1 - [test_normalization.py]=1 # probably control flow (uninitialized variable) - [test_nntplib.py]=1 + [test_nntplib.py]=1 # Too long in running before decomplation takes 25 seconds [test_optparse.py]=1 # doesn't terminate at test_consume_separator_stop_at_option - [test_os.py]=1 # probably control flow (uninitialized variable) [test_ossaudiodev.py]=1 # it fails on its own [test_pdb.py]=1 # Probably relies on comments [test_peepholer.py]=1 # test assert error - [test_pickle.py]=1 # Probably relies on comments - [test_poll.py]=1 - [test_poplib.py]=1 - [test_pydoc.py]=1 # it fails on its own - [test_runpy.py]=1 # [test_pkg.py]=1 # Investigate: lists differ [test_pkgutil.py]=1 # Investigate: - [test_platform.py]=1 # probably control flow: uninitialized variable - [test_pow.py]=1 # probably control flow: test assertion failure + [test_poll.py]=1 # Takes too long to run before decompiling 11 seconds [test_pwd.py]=1 # killing - doesn't terminate + [test_pydoc.py]=1 # it fails on its own [test_regrtest.py]=1 # lists differ - [test_re.py]=1 # test assertion error [test_richcmp.py]=1 # parse error + [test_runpy.py]=1 # Too long to run before decompiling [test_select.py]=1 # test takes too long to run: 11 seconds - [test_selectors.py]=1 + [test_selectors.py]=1 # Takes too long to run before decompling: 17 seconds [test_shutil.py]=1 # fails on its own - [test_signal.py]=1 # - [test_slice.py]=1 # test assert error in data; Investigate - [test_smtplib.py]=1 # - [test_socket.py]=1 - [test_socketserver.py]=1 - [test_sort.py]=1 # Probably control flow; unintialized varaible - [test_ssl.py]=1 # Probably control flow; unintialized varaible + [test_signal.py]=1 # Takes too long to run before decompiling: 22 seconds + [test_smtplib.py]=1 # test failures + [test_socket.py]=1 # Takes too long to run before decompiling + [test_ssl.py]=1 # Takes too long to run more than 15 seconds. Probably control flow; unintialized variable [test_startfile.py]=1 # it fails on its own - [test_statistics.py]=1 # Probably control flow; unintialized varaible - [test_string_literals.py]=1 # Investigate boolean parsing - [test_strptime.py]=1 # test assertions failed + [test_strptime.py]=1 # parfse error [test_strtod.py]=1 # test assertions failed - [test_structmembers.py]=1 # test assertions failed - [test_struct.py]=1 # test assertions failed - [test_subprocess.py]=1 - [test_sys_setprofile.py]=1 # test assertions failed + [test_struct.py]=1 # probably control flow + [test_subprocess.py]=1 # Takes too long to run before decompile: 25 seconds [test_sys_settrace.py]=1 # parse error - [test_sysconfig.py]=1 # if confused for ifelse in "test_triplet_in_ext_suffix" [test_tarfile.py]=1 # test assertions failed - [test_threading.py]=1 # - [test_timeit.py]=1 # probably control flow uninitialized variable + [test_threading.py]=1 # test assertion failers [test_tk.py]=1 # test takes too long to run: 13 seconds - [test_tokenize.py]=1 + [test_tokenize.py]=1 # test takes too long to run before decompilation: 43 seconds [test_trace.py]=1 # it fails on its own [test_traceback.py]=1 # Probably uses comment for testing - [test_tracemalloc.py]=1 # + [test_tracemalloc.py]=1 # test assert failres + [test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds [test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds [test_typing.py]=1 # parse error [test_types.py]=1 # parse error [test_unicode.py]=1 # unicode thing - [test_urllib2_localnet.py]=1 # [test_urllibnet.py]=1 # probably control flow - uninitialized variable [test_weakref.py]=1 # probably control flow - uninitialized variable [test_with.py]=1 # probably control flow - uninitialized variable + [test_winconsoleio.py]=1 # it fails on its own [test_winreg.py]=1 # it fails on its own [test_winsound.py]=1 # it fails on its own @@ -141,11 +131,18 @@ SKIP_TESTS=( [test_zipfile.py]=1 # it fails on its own [test_zipfile64.py]=1 # Too long to run ) -# 282 unit-test files in about 19 minutes +# 306 unit-test files in about 19 minutes if (( batch )) ; then + SKIP_TESTS[test_dbm_gnu.py]=1 # fails on its own on POWER SKIP_TESTS[test_distutils.py]=1 SKIP_TESTS[test_fileio.py]=1 SKIP_TESTS[test_gc.py]=1 + SKIP_TESTS[test_idle.py]=1 # Probably installation specific + SKIP_TESTS[test_sqlite.py]=1 # fails on its own on POWER + SKIP_TESTS[test_tix.py]=1 # it fails on its own + SKIP_TESTS[test_ttk_textonly.py]=1 # Installation dependent? + SKIP_TESTS[test_venv.py]=1 # Too long to run: 11 seconds SKIP_TESTS[test_zipimport_support.py]=1 + fi diff --git a/test/stdlib/3.8-exclude.sh b/test/stdlib/3.8-exclude.sh index 3ada5b24..8eaf5254 100644 --- a/test/stdlib/3.8-exclude.sh +++ b/test/stdlib/3.8-exclude.sh @@ -1,166 +1,337 @@ SKIP_TESTS=( + [test_time.py]=1 # FIXME: works on uncompyle6? + [test_urllib2.py]=1 # FIXME: works on uncompyle6? + [test_zipimport.py]=1 # FIXME: works on uncompyle6 + [test___all__.py]=1 # it fails on its own [test_aifc.py]=1 # parse error [test_argparse.py]=1 #- it fails on its own [test_array.py]=1 #- parse error [test_asdl_parser.py]=1 # it fails on its own [test_ast.py]=1 # Depends on comments in code + [test_asyncgen.py]=1 # parse error + [test_asynchat.py]=1 # parse error + [test_asyncore.py]=1 # parse error [test_atexit.py]=1 # The atexit test looks for specific comments in error lines + [test_audioop.py]=1 # test failure + [test_audit.py]=1 # parse error + [test_base64.py]=1 # parse error [test_baseexception.py]=1 # + [test_bigaddrspace.py]=1 # parse error + [test_bigmem.py]=1 # parse error [test_bdb.py]=1 # + [test_binascii.py]=1 # test failure + [test_binhex.py]=1 # parse error + [test_binop.py]=1 # parse error + [test_bool.py]=1 # parse error [test_buffer.py]=1 # parse error [test_builtin.py]=1 # parse error + [test_bytes.py]=1 # parse error + [test_bz2.py]=1 # parse error + [test_calendar.py]=1 # parse error + [test_cgi.py]=1 # parse error + [test_cgitb.py]=1 # parse error [test_clinic.py]=1 # it fails on its own [test_cmath.py]=1 # test assertion failure + [test_cmd.py]=1 # parse error [test_cmd_line.py]=1 # Interactive? [test_cmd_line_script.py]=1 + [test_code_module.py]=1 # test failure + [test_codecmaps_cn.py]=1 # test before decompile takes too long to run 135 secs + [test_codecmaps_hk.py]=1 # test before decompile takes too long to run 46 secs [test_codecs.py]=1 [test_collections.py]=1 [test_compare.py]=1 [test_compile.py]=1 + [test_compileall.py]=1 # fails on its own + [test_complex.py]=1 # Investigate [test_concurrent_futures.py]=1 # too long [test_configparser.py]=1 [test_context.py]=1 + [test_contextlib.py]=1 # parse error + [test_contextlib_async.py]=1 # parse error [test_coroutines.py]=1 # Parse error - [test_codecs.py]=1 - [test_compile.py]=1 # Parse error, but after that, the code introspects on co_consts in a non-decompilable way - [test_complex.py]=1 # Investigate + [test_cprofile.py]=1 # parse error [test_crypt.py]=1 # Parse error + [test_csv.py]=1 # Parse error [test_ctypes.py]=1 # it fails on its own [test_curses.py]=1 # Parse error [test_dataclasses.py]=1 # test assertion errors [test_datetime.py]=1 # Takes too long + [test_dbm.py]=1 # parse error + [test_dbm_dumb.py]=1 # parse error [test_dbm_gnu.py]=1 # Takes too long [test_dbm_ndbm.py]=1 # it fails on its own [test_decimal.py]=1 # Parse error + [test_decorators.py]=1 # parse error + [test_deque.py]=1 # parse error [test_descr.py]=1 # Parse error + [test_descrtut.py]=1 # parse error [test_devpoll.py]=1 # it fails on its own + [test_dict.py]=1 # parse error [test_dictcomps.py]=1 # Bad semantics - Investigate + [test_difflib.py]=1 # parse error [test_dis.py]=1 # We change line numbers - duh! + [test_doctest.py]=1 # parse error [test_docxmlrpc.py]=1 + [test_dtrace.py]=1 # parse error + [test_dummy_thread.py]=1 # parse error - [test_exceptions.py]=1 # parse error - [test_enumerate.py]=1 # + [test_embed.py]=1 # parse error + [test_ensureip.py]=1 # + [test_ensurepip.py]=1 # parse error [test_enum.py]=1 # + [test_enumerate.py]=1 # + [test_eof.py]=1 # parse error + [test_epoll.py]=1 # parse error + [test_exception_hierarchy.py]=1 # control flow? + [test_exceptions.py]=1 # parse error [test_faulthandler.py]=1 # takes too long [test_fcntl.py]=1 + [test_filecmp.py]=1 # parse error [test_fileinput.py]=1 - [test_float.py]=1 + [test_fileio.py]=1 + [test_float.py]=1 # Takes a long time to decompile + [test_flufl.py]=1 # parse error [test_format.py]=1 [test_frame.py]=1 + [test_frozen.py]=1 # parse error [test_fstring.py]=1 # Investigate [test_ftplib.py]=1 [test_functools.py]=1 + [test_future.py]=1 # parse error + [test___future__.py]=1 # test failure + [test_future5.py]=1 # parse error + [test_gc.py]=1 # parse error [test_gdb.py]=1 # it fails on its own + [test_genericpath.py]=1 # parse error [test_generators.py]=1 # improper decompile of assert i < n and (n-i) % 3 == 0 + [test_getpass.py]=1 # parse error + [test_gettext.py]=1 # parse error [test_glob.py]=1 # [test_grammar.py]=1 [test_grp.py]=1 # Doesn't terminate (killed) [test_gzip.py]=1 # parse error [test_hashlib.py]=1 # test assert failures + [test_heapq.py]=1 # test failure + [test_hmac.py]=1 # parse error [test_httplib.py]=1 # parse error [test_http_cookiejar.py]=1 - [test_imaplib-3.7.py]=1 - [test_idle.py]=1 # Probably installation specific + [test_httpservers.py]=1 # parse error + + [test_imghdr.py]=1 # parse error + [test_imp.py]=1 # parse error + [test_int.py]=1 # parse error [test_io.py]=1 # test takes too long to run: 37 seconds + [test_ioctl.py]=1 # parse error [test_imaplib.py]=1 + [test_ipaddress.py]=1 # parse error [test_index.py]=1 [test_inspect.py]=1 + [test_iter.py]=1 # parse error [test_itertools.py]=1 # parse error [test_keywordonlyarg.py]=1 # parse error [test_kqueue.py]=1 # it fails on its own + [test__locale.py]=1 # parse error + [test_largefile.py]=1 # parse error [test_lib2to3.py]=1 # it fails on its own - [test_long.py]=1 # investigate + [test_linecache.py]=1 # parse error + [test_lltrace.py]=1 # parse error + [test_locale.py]=1 # parse error [test_logging.py]=1 # test takes too long to run: 20 seconds + [test_long.py]=1 # investigate + [test_lzma.py]=1 # it fails on its own - [test_mailbox.py]=1 + [test_mailbox.py]=1 # parse error + [test_mailcap.py]=1 # parse error [test_marshal.py]=1 [test_math.py]=1 + [test_memoryio.py]=1 # test failure + [test_memoryview.py]=1 # parse error + [test_minidom.py]=1 # test failure + [test_mmap.py]=1 # parse error [test_modulefinder.py]=1 [test_msilib.py]=1 [test_multiprocessing_fork.py]=1 # test takes too long to run: 62 seconds [test_multiprocessing_forkserver.py]=1 + [test_multiprocessing_main_handling.py]=1 # parse error [test_multiprocessing_spawn.py]=1 + [test_named_expressions.py]=1 # parse error + [test_netrc.py]=1 # parse error + [test_nis.py]=1 # break outside of loop [test_normalization.py]=1 # probably control flow (uninitialized variable) [test_nntplib.py]=1 + [test_ntpath.py]=1 + [test__osx_support.py]=1 # parse error + [test_opcodes.py]=1 # parse error + [test_operator.py]=1 # parse error [test_optparse.py]=1 # doesn't terminate (killed) + [test_ordered_dict.py]=1 # parse error [test_os.py]=1 # probably control flow (uninitialized variable) [test_ossaudiodev.py]=1 # it fails on its own + [test_osx_env.py]=1 # parse error [test_pathlib.py]=1 # parse error [test_pdb.py]=1 # Probably relies on comments [test_peepholer.py]=1 # decompile takes a long time; then test assert error [test_pickle.py]=1 # Probably relies on comments - [test_poll.py]=1 - [test_poplib.py]=1 - [test_pydoc.py]=1 # it fails on its own - [test_runpy.py]=1 # - [test_pkg.py]=1 # parse error; Investigate: lists differ - [test_pkgutil.py]=1 # parse error + [test_picklebuffer.py]=1 # parse error + [test_pipes.py]=1 # parse error + [test_pkg.py]=1 # Investigate: lists differ + [test_pkgimport.py]=1 # parse error + [test_pkgutil.py]=1 # Investigate: [test_platform.py]=1 # parse error + [test_plistlib.py]=1 # parse error + [test_poll.py]=1 + [test_popen.py]=1 # parse error + [test_poplib.py]=1 # Parse error + [test_positional_only_arg.py]=1 # test failures + [test_posixpath.py]=1 # parse error + [test_posix.py]=1 # parse error + [test_print.py]=1 # parse error + [test_profile.py]=1 # parse error [test_pwd.py]=1 # killing - doesn't terminate + [test_pulldom.py]=1 # killing - doesn't terminate + [test_py_compile.py]=1 # parse error + [test_pyexpat.py]=1 # parse error + [test_pyclbr.py]=1 # test failure + [test_pydoc.py]=1 # it fails on its own - [test_regrtest.py]=1 # parse error; test assertion error: lists differ - [test_re.py]=1 # parse error; test assertion error - [test_richcmp.py]=1 # Investigate: data[i] index error in semantic handling + [test_queue.py]=1 # parse error + [test_raise.py]=1 # parse error + [test_random.py]=1 # parse error + [test_range.py]=1 # parse error + [test_rcompleter.py]=1 # parse error + [test_re.py]=1 # test assertion error + [test_readline.py]=1 # parse error + [test_robotparser.py]=1 # too long to run before decompiling: 31 secs + [test_regrtest.py]=1 # lists differ + [test_reprlib.py]=1 # parse error + [test_resource.py]=1 # parse error + [test_richcmp.py]=1 # parse error + [test_runpy.py]=1 # + + [test_sax.py]=1 # parse error + [test_sched.py]=1 # parse error + [test_scope.py]=1 # parse error + [test_script_helper.py]=1 # parse error [test_select.py]=1 # test takes too long to run: 11 seconds [test_selectors.py]=1 + [test_set.py]=1 # parse error + [test_shelve.py]=1 # parse error + [test_shlex.py]=1 # probably control flow [test_shutil.py]=1 # fails on its own [test_signal.py]=1 # - [test_slice.py]=1 # Investigate: test assertion error + [test_site.py]=1 # parse error + [test_slice.py]=1 # Investigate + [test_smtpd.py]=1 # parse error [test_smtplib.py]=1 # + [test_smtpnet.py]=1 # parse error [test_socket.py]=1 [test_socketserver.py]=1 - [test_sort.py]=1 # parse error; - [test_ssl.py]=1 # parse error + [test_sort.py]=1 # Probably control flow; unintialized varaible + [test_source_encoding.py]=1 # parse error + [test_spwd.py]=1 # parse error + [test_ssl.py]=1 # Probably control flow; unintialized varaible [test_startfile.py]=1 # it fails on its own - [test_statistics.py]=1 # Takes more than 15 secs to run. Assert failures - [test_stat.py]=1 # parse error; test assertions failed - [test_string_literals.py]=1 # parse error; Investigate boolean parsing + [test_stat.py]=1 # test assertions failed + [test_statistics.py]=1 # Probably control flow; unintialized varaible + [test_strftime.py]=1 # parse error + [test_string.py]=1 # parse error + [test_string_literals.py]=1 # parse error [test_strptime.py]=1 # test assertions failed [test_strtod.py]=1 # test assertions failed - [test_structmembers.py]=1 # test assertions failed [test_struct.py]=1 # test assertions failed + [test_structmembers.py]=1 # test assertions failed + [test_subclassinit.py]=1 # parse error [test_subprocess.py]=1 + [test_super.py]=1 # parse error + [test_support.py]=1 # parse error + [test_symbol.py]=1 # parse error + [test_sys.py]=1 # parse error [test_sys_setprofile.py]=1 # test assertions failed [test_sys_settrace.py]=1 # parse error - [test_sysconfig.py]=1 # if confused for ifelse in "test_triplet_in_ext_suffix" + [test_sysconfig.py]=1 # parse error + [test_tabnanny.py]=1 # parse error [test_tarfile.py]=1 # parse error + [test_tcl.py]=1 # parse error + [test_telnetlib.py]=1 # parse error + [test_tempfile.py]=1 # parse error + [test_thread.py]=1 # parse error + [test_threaded_import.py]=1 # parse error [test_threading.py]=1 # [test_timeit.py]=1 # probably control flow uninitialized variable + [test_timeout.py]=1 # parse error [test_tk.py]=1 # test takes too long to run: 13 seconds [test_tokenize.py]=1 [test_trace.py]=1 # it fails on its own [test_traceback.py]=1 # Probably uses comment for testing [test_tracemalloc.py]=1 # [test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds - [test_typing.py]=1 # parse error + [test_turtle_import.py]=1 # parse error + [test_turtle.py]=1 # parse error [test_types.py]=1 # parse error + [test_typing.py]=1 # parse error + [test_ucn.py]=1 # parse error [test_unicode.py]=1 # unicode thing + [test_unicode_file_functions.py]=1 # parse faiure + [test_unicodedata.py]=1 # test faiure + [test_univnewlines.py]=1 # parse error + [test_urllib.py]=1 # parse error [test_urllib2.py]=1 # + [test_urllib_response.py]=1 # parse error [test_urllib2_localnet.py]=1 # + [test_urllib2net.py]=1 # parse error [test_urllibnet.py]=1 # probably control flow - uninitialized variable + [test_urlparse.py]=1 # parse error + [test_userdict.py]=1 # test failures + [test_userstring.py]=1 # parse error + [test_utf8.py]=1 # parse error + [test_utf8_mode.py]=1 # parse error + [test_uu.py]=1 # parse error + [test_uuid.py]=1 # parse error + + [test_venv.py]=1 # parse error [test_weakref.py]=1 # probably control flow - uninitialized variable + [test_weakset.py]=1 # parse error + [test_webbrowser.py]=1 # parse error [test_with.py]=1 # probably control flow - uninitialized variable [test_winconsoleio.py]=1 # it fails on its own [test_winreg.py]=1 # it fails on its own [test_winsound.py]=1 # it fails on its own + [test_wsgiref.py]=1 # parse error + [test_xml_etree.py]=1 # parse error + [test_xmlrpc.py]=1 # parse error + [test__xxsubinterpreters.py]=1 # parse error + + [test_yield_from.py]=1 # parse error + + [test_zlib.py]=1 # test looping take more than 15 seconds to run + [test_zipapp.py]=1 # parse error + [test_zipimport_support.py]=1 # parse error [test_zipfile.py]=1 # it fails on its own [test_zipfile64.py]=1 # ) -# 268 About unit-test files, in about 11 minutes +# 114 test files, Elapsed time about 7 minutes + +if (( batch )) ; then + SKIP_TESTS[test_idle.py]=1 # Probably installation specific + SKIP_TESTS[test_tix.py]=1 # fails on its own + SKIP_TESTS[test_ttk_textonly.py]=1 # Installation dependent? + +fi diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 66ec2b7e..1ab6e7c8 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -568,14 +568,14 @@ class PythonParser(GenericASTBuilder): _mklambda ::= mklambda - expr ::= conditional + expr ::= if_exp ret_expr ::= expr ret_expr ::= ret_and ret_expr ::= ret_or ret_expr_or_cond ::= ret_expr - ret_expr_or_cond ::= ret_cond + ret_expr_or_cond ::= if_exp_ret stmt ::= return_lambda diff --git a/uncompyle6/parsers/parse23.py b/uncompyle6/parsers/parse23.py index 6e29d9e8..166de860 100644 --- a/uncompyle6/parsers/parse23.py +++ b/uncompyle6/parsers/parse23.py @@ -49,7 +49,7 @@ class Python23Parser(Python24Parser): and2 ::= _jump jmp_false COME_FROM expr COME_FROM alias ::= IMPORT_NAME attributes store - conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM + if_exp ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM ''' def customize_grammar_rules(self, tokens, customize): diff --git a/uncompyle6/parsers/parse24.py b/uncompyle6/parsers/parse24.py index 7795b7ae..9a804ab6 100644 --- a/uncompyle6/parsers/parse24.py +++ b/uncompyle6/parsers/parse24.py @@ -58,7 +58,7 @@ class Python24Parser(Python25Parser): def remove_rules_24(self): self.remove_rules(""" - expr ::= conditional + expr ::= if_exp """) diff --git a/uncompyle6/parsers/parse25.py b/uncompyle6/parsers/parse25.py index 1e5c0e61..62b27289 100644 --- a/uncompyle6/parsers/parse25.py +++ b/uncompyle6/parsers/parse25.py @@ -80,7 +80,7 @@ class Python25Parser(Python26Parser): classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1 classdefdeco2 ::= LOAD_CONST expr mkfunc CALL_FUNCTION_0 BUILD_CLASS kv3 ::= expr expr STORE_MAP - ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond + if_exp_ret ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP return_if_stmts ::= return_if_stmt @@ -89,13 +89,12 @@ class Python25Parser(Python26Parser): return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA setupwithas ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 setup_finally stmt ::= classdefdeco - stmt ::= if_expr_lambda - stmt ::= conditional_not_lambda - if_expr_lambda ::= expr jmp_false_then expr return_if_lambda - return_stmt_lambda LAMBDA_MARKER - conditional_not_lambda - ::= expr jmp_true_then expr return_if_lambda + stmt ::= if_exp_lambda + stmt ::= if_exp_not_lambda + if_exp_lambda ::= expr jmp_false_then expr return_if_lambda return_stmt_lambda LAMBDA_MARKER + if_exp_not_lambda ::= expr jmp_true_then expr return_if_lambda + return_stmt_lambda LAMBDA_MARKER """) super(Python25Parser, self).customize_grammar_rules(tokens, customize) if self.version == 2.5: diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index 383966b2..7a7f2271 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -262,8 +262,8 @@ class Python26Parser(Python2Parser): ''' ret_and ::= expr jmp_false ret_expr_or_cond COME_FROM ret_or ::= expr jmp_true ret_expr_or_cond COME_FROM - ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond - ret_cond ::= expr jmp_false_then expr ret_expr_or_cond + if_exp_ret ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond + if_exp_ret ::= expr jmp_false_then expr ret_expr_or_cond return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP return ::= ret_expr RETURN_VALUE POP_TOP @@ -283,11 +283,11 @@ class Python26Parser(Python2Parser): kvlist ::= kvlist kv3 # Note: preserve positions 0 2 and 4 for semantic actions - conditional_not ::= expr jmp_true expr jf_cf_pop expr COME_FROM - conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt - conditional ::= expr jmp_false expr ja_cf_pop expr + if_exp_not ::= expr jmp_true expr jf_cf_pop expr COME_FROM + if_exp ::= expr jmp_false expr jf_cf_pop expr come_from_opt + if_exp ::= expr jmp_false expr ja_cf_pop expr - expr ::= conditional_not + expr ::= if_exp_not and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP @@ -311,27 +311,27 @@ class Python26Parser(Python2Parser): compare_chained2 ::= expr COMPARE_OP return_lambda return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP - stmt ::= if_expr_lambda - stmt ::= conditional_not_lambda - if_expr_lambda ::= expr jmp_false_then expr return_if_lambda + stmt ::= if_exp_lambda + stmt ::= if_exp_not_lambda + if_exp_lambda ::= expr jmp_false_then expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - conditional_not_lambda ::= + if_exp_not_lambda ::= expr jmp_true_then expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - # if_expr_true are for conditions which always evaluate true + # if_exp_true are for conditions which always evaluate true # There is dead or non-optional remnants of the condition code though, # and we use that to match on to reconstruct the source more accurately - expr ::= if_expr_true - if_expr_true ::= expr jf_pop expr COME_FROM + expr ::= if_exp_true + if_exp_true ::= expr jf_pop expr COME_FROM # This comes from # 0 or max(5, 3) if 0 else 3 # where there seems to be an additional COME_FROM at the # end. Not sure if this is appropriately named or # is the best way to handle - expr ::= conditional_false - conditional_false ::= conditional COME_FROM + expr ::= if_exp_false + if_exp_false ::= if_exp COME_FROM """ @@ -366,10 +366,10 @@ class Python26Parser(Python2Parser): if ast[1] is None: return False - # For now, we won't let the 2nd 'expr' be a "conditional_not" + # For now, we won't let the 2nd 'expr' be a "if_exp_not" # However in < 2.6 where we don't have if/else expression it *can* # be. - if self.version >= 2.6 and ast[2][0] == 'conditional_not': + if self.version >= 2.6 and ast[2][0] == "if_exp_not": return True test_index = last diff --git a/uncompyle6/parsers/parse27.py b/uncompyle6/parsers/parse27.py index 24d61a83..e31a4730 100644 --- a/uncompyle6/parsers/parse27.py +++ b/uncompyle6/parsers/parse27.py @@ -97,9 +97,9 @@ class Python27Parser(Python2Parser): jmp_false ::= POP_JUMP_IF_FALSE jmp_true ::= POP_JUMP_IF_TRUE - ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM - ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM - ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond + ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM + ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM + if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM @@ -116,17 +116,17 @@ class Python27Parser(Python2Parser): compare_chained2 ::= expr COMPARE_OP return_lambda compare_chained2 ::= expr COMPARE_OP return_lambda - # if_expr_true are for conditions which always evaluate true + # if_exp_true are for conditions which always evaluate true # There is dead or non-optional remnants of the condition code though, # and we use that to match on to reconstruct the source more accurately. # FIXME: we should do analysis and reduce *only* if there is dead code? # right now we check that expr is "or". Any other nodes types? - expr ::= if_expr_true - if_expr_true ::= expr JUMP_FORWARD expr COME_FROM + expr ::= if_exp_true + if_exp_true ::= expr JUMP_FORWARD expr COME_FROM - conditional ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM - conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr + if_exp ::= expr jmp_false expr JUMP_FORWARD expr COME_FROM + if_exp ::= expr jmp_false expr JUMP_ABSOLUTE expr """ def p_stmt27(self, args): @@ -185,21 +185,24 @@ class Python27Parser(Python2Parser): ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel ifelsestmtl ::= testexpr c_stmts_opt CONTINUE else_suitel + # In the future when we have ifelsestmtl checking we should add something like: + # ifelsestmtl ::= testexpr c_stmts_opt JUMP_FORWARD else_suite come_froms + # c_stmts ::= ifelsestmtl + # "if"/"else" statement that ends in a RETURN ifelsestmtr ::= testexpr return_if_stmts COME_FROM returns # Common with 2.6 return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM - stmt ::= if_expr_lambda - stmt ::= conditional_not_lambda - if_expr_lambda ::= expr jmp_false expr return_if_lambda + stmt ::= if_exp_lambda + stmt ::= if_exp_not_lambda + if_exp_lambda ::= expr jmp_false expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - conditional_not_lambda - ::= expr jmp_true expr return_if_lambda + if_exp_not_lambda ::= expr jmp_true expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - expr ::= conditional_not - conditional_not ::= expr jmp_true expr _jump expr COME_FROM + expr ::= if_exp_not + if_exp_not ::= expr jmp_true expr _jump expr COME_FROM kv3 ::= expr expr STORE_MAP """ @@ -223,13 +226,13 @@ class Python27Parser(Python2Parser): # FIXME: Put more in this table self.reduce_check_table = { - "ifelsestmt": ifelsestmt, + # "ifelsestmt": ifelsestmt, "tryelsestmt": tryelsestmt, "tryelsestmtl": tryelsestmt, } self.check_reduce["and"] = "AST" - self.check_reduce["conditional"] = "AST" + self.check_reduce["if_exp"] = "AST" self.check_reduce["except_handler"] = "tokens" self.check_reduce["except_handler_else"] = "tokens" @@ -241,7 +244,7 @@ class Python27Parser(Python2Parser): self.check_reduce["list_if_not"] = "AST" self.check_reduce["list_if"] = "AST" self.check_reduce["comp_if"] = "AST" - self.check_reduce["if_expr_true"] = "tokens" + self.check_reduce["if_exp_true"] = "tokens" self.check_reduce["whilestmt"] = "tokens" return @@ -266,7 +269,7 @@ class Python27Parser(Python2Parser): return tokens[first].offset < jmp_false[0].attr < tokens[last].offset pass elif (rule[0], rule[1][0:5]) == ( - "conditional", + "if_exp", ("expr", "jmp_false", "expr", "JUMP_ABSOLUTE", "expr")): jmp_false = ast[1] if jmp_false[0] == "POP_JUMP_IF_FALSE": @@ -336,7 +339,7 @@ class Python27Parser(Python2Parser): while (tokens[i] != "JUMP_BACK"): i -= 1 return tokens[i].attr != tokens[i-1].attr - elif rule[0] == "if_expr_true": + elif rule[0] == "if_exp_true": return (first) > 0 and tokens[first-1] == "POP_JUMP_IF_FALSE" return False diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 11e3fe13..76894d23 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -32,8 +32,10 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parsers.reducecheck import ( and_check, except_handler_else, + ifelsestmt, ifstmt, iflaststmt, + or_check, testtrue, tryelsestmtl3, tryexcept, @@ -332,9 +334,9 @@ class Python3Parser(PythonParser): jmp_true ::= POP_JUMP_IF_TRUE # FIXME: Common with 2.7 - ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM - ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM - ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond + ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM + ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM + if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM or ::= expr jmp_true expr @@ -349,13 +351,12 @@ class Python3Parser(PythonParser): def p_stmt3(self, args): """ - stmt ::= if_expr_lambda + stmt ::= if_exp_lambda - stmt ::= conditional_not_lambda - if_expr_lambda ::= expr jmp_false expr return_if_lambda + stmt ::= if_exp_not_lambda + if_exp_lambda ::= expr jmp_false expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - conditional_not_lambda - ::= expr jmp_true expr return_if_lambda + if_exp_not_lambda ::= expr jmp_true expr return_if_lambda return_stmt_lambda LAMBDA_MARKER return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA @@ -469,18 +470,18 @@ class Python3Parser(PythonParser): def p_expr3(self, args): """ expr ::= LOAD_STR - expr ::= conditionalnot - conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM + expr ::= if_exp_not + if_exp_not ::= expr jmp_true expr jump_forward_else expr COME_FROM # a JUMP_FORWARD to another JUMP_FORWARD can get turned into # a JUMP_ABSOLUTE with no COME_FROM - conditional ::= expr jmp_false expr jump_absolute_else expr + if_exp ::= expr jmp_false expr jump_absolute_else expr - # if_expr_true are for conditions which always evaluate true + # if_exp_true are for conditions which always evaluate true # There is dead or non-optional remnants of the condition code though, # and we use that to match on to reconstruct the source more accurately - expr ::= if_expr_true - if_expr_true ::= expr JUMP_FORWARD expr COME_FROM + expr ::= if_exp_true + if_exp_true ::= expr JUMP_FORWARD expr COME_FROM """ @staticmethod @@ -707,12 +708,11 @@ class Python3Parser(PythonParser): stmt ::= assign2_pypy assign3_pypy ::= expr expr expr store store store assign2_pypy ::= expr expr store store - stmt ::= if_expr_lambda - stmt ::= conditional_not_lambda + stmt ::= if_exp_lambda + stmt ::= if_exp_not_lambda if_expr_lambda ::= expr jmp_false expr return_if_lambda return_lambda LAMBDA_MARKER - conditional_not_lambda - ::= expr jmp_true expr return_if_lambda + if_exp_not_lambda ::= expr jmp_true expr return_if_lambda return_lambda LAMBDA_MARKER """, nop_func, @@ -1535,6 +1535,10 @@ class Python3Parser(PythonParser): self.reduce_check_table = { "except_handler_else": except_handler_else, # "ifstmt": ifstmt, + "ifstmtl": ifstmt, + "ifelsestmtc": ifelsestmt, + "ifelsestmt": ifelsestmt, + "or": or_check, "testtrue": testtrue, "tryelsestmtl3": tryelsestmtl3, "try_except": tryexcept, @@ -1544,18 +1548,19 @@ class Python3Parser(PythonParser): self.reduce_check_table["and"] = and_check self.check_reduce["and"] = "AST" + self.check_reduce["annotate_tuple"] = "noAST" self.check_reduce["aug_assign1"] = "AST" self.check_reduce["aug_assign2"] = "AST" - self.check_reduce["while1stmt"] = "noAST" - self.check_reduce["while1elsestmt"] = "noAST" + self.check_reduce["except_handler_else"] = "tokens" self.check_reduce["ifelsestmt"] = "AST" + self.check_reduce["ifelsestmtc"] = "AST" self.check_reduce["ifstmt"] = "AST" + self.check_reduce["ifstmtl"] = "AST" if self.version == 3.6: self.reduce_check_table["iflaststmtl"] = iflaststmt self.check_reduce["iflaststmt"] = "AST" self.check_reduce["iflaststmtl"] = "AST" - self.check_reduce["annotate_tuple"] = "noAST" - self.check_reduce["except_handler_else"] = "tokens" + self.check_reduce["or"] = "AST" self.check_reduce["testtrue"] = "tokens" if not PYTHON3: self.check_reduce["kwarg"] = "noAST" @@ -1565,8 +1570,8 @@ class Python3Parser(PythonParser): self.check_reduce["try_except"] = "AST" self.check_reduce["tryelsestmtl3"] = "AST" - # FIXME: remove parser errors caused by the below - # self.check_reduce['while1elsestmt'] = 'noAST' + self.check_reduce["while1stmt"] = "noAST" + self.check_reduce["while1elsestmt"] = "noAST" return def reduce_is_invalid(self, rule, ast, tokens, first, last): diff --git a/uncompyle6/parsers/parse30.py b/uncompyle6/parsers/parse30.py index 53c11ada..7a17d3da 100644 --- a/uncompyle6/parsers/parse30.py +++ b/uncompyle6/parsers/parse30.py @@ -262,9 +262,9 @@ class Python30Parser(Python31Parser): compare_chained2 COME_FROM ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM - ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF + if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond - ret_expr_or_cond ::= ret_cond + ret_expr_or_cond ::= if_exp_ret or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py index 800fa38d..19acf82d 100644 --- a/uncompyle6/parsers/parse32.py +++ b/uncompyle6/parsers/parse32.py @@ -15,7 +15,7 @@ class Python32Parser(Python3Parser): def p_32to35(self, args): """ - conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM + if_exp ::= expr jmp_false expr jump_forward_else expr COME_FROM # compare_chained2 is used in a "chained_compare": x <= y <= z # used exclusively in compare_chained diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index ffc40115..111569ad 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -170,26 +170,33 @@ class Python35Parser(Python34Parser): elif opname == 'BEFORE_ASYNC_WITH' and self.version < 3.8: # Some Python 3.5+ async additions rules_str = """ - async_with_stmt ::= expr - stmt ::= async_with_stmt + async_with_stmt ::= expr + stmt ::= async_with_stmt + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH + async_with_post ::= COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY - async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + async_with_post - stmt ::= async_with_as_stmt + stmt ::= async_with_as_stmt async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY + async_with_pre + store + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post """ self.addRule(rules_str, nop_func) elif opname == 'BUILD_MAP_UNPACK': diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 0321e665..815b9cdd 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -64,7 +64,7 @@ class Python36Parser(Python35Parser): jf_cf ::= JUMP_FORWARD COME_FROM cf_jf_else ::= come_froms JUMP_FORWARD ELSE - conditional ::= expr jmp_false expr jf_cf expr COME_FROM + if_exp ::= expr jmp_false expr jf_cf expr COME_FROM async_for_stmt ::= SETUP_LOOP expr GET_AITER @@ -112,9 +112,16 @@ class Python36Parser(Python35Parser): except_suite ::= c_stmts_opt COME_FROM POP_EXCEPT jump_except COME_FROM jb_cfs ::= JUMP_BACK come_froms + + # If statement inside a loop. + stmt ::= ifstmtl + ifstmtl ::= testexpr _ifstmts_jumpl + _ifstmts_jumpl ::= c_stmts JUMP_BACK + ifelsestmtl ::= testexpr c_stmts_opt jb_cfs else_suitel ifelsestmtl ::= testexpr c_stmts_opt cf_jf_else else_suitel - ifelsestmt ::= testexpr c_stmts_opt cf_jf_else else_suite _come_froms + ifelsestmt ::= testexpr c_stmts_opt cf_jf_else else_suite _come_froms + ifelsestmt ::= testexpr c_stmts come_froms else_suite come_froms # In 3.6+, A sequence of statements ending in a RETURN can cause # JUMP_FORWARD END_FINALLY to be omitted from try middle @@ -157,8 +164,8 @@ class Python36Parser(Python35Parser): # that and then we can remove this. def p_37conditionals(self, args): """ - expr ::= conditional37 - conditional37 ::= expr expr jf_cfs expr COME_FROM + expr ::= if_exp37 + if_exp37 ::= expr expr jf_cfs expr COME_FROM jf_cfs ::= JUMP_FORWARD _come_froms ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except """ @@ -168,6 +175,8 @@ class Python36Parser(Python35Parser): # """) super(Python36Parser, self).customize_grammar_rules(tokens, customize) self.remove_rules(""" + _ifstmts_jumpl ::= c_stmts_opt + _ifstmts_jumpl ::= _ifstmts_jump except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM async_for_stmt ::= SETUP_LOOP expr GET_AITER @@ -230,24 +239,26 @@ class Python36Parser(Python35Parser): elif opname == 'BEFORE_ASYNC_WITH': rules_str = """ stmt ::= async_with_stmt + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH + async_with_post ::= COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store + async_with_pre + store suite_stmts_opt POP_BLOCK LOAD_CONST - COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY + async_with_post stmt ::= async_with_as_stmt async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts_opt + POP_TOP + suite_stmts_opt POP_BLOCK LOAD_CONST - COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY + async_with_post + async_with_stmt ::= expr + POP_TOP + suite_stmts_opt + async_with_post """ self.addRule(rules_str, nop_func) diff --git a/uncompyle6/parsers/parse37.py b/uncompyle6/parsers/parse37.py index 9cca1af0..84d29502 100644 --- a/uncompyle6/parsers/parse37.py +++ b/uncompyle6/parsers/parse37.py @@ -185,14 +185,14 @@ class Python37Parser(Python37BaseParser): _mklambda ::= mklambda - expr ::= conditional + expr ::= if_exp - ret_expr ::= expr - ret_expr ::= ret_and - ret_expr ::= ret_or + ret_expr ::= expr + ret_expr ::= ret_and + ret_expr ::= ret_or ret_expr_or_cond ::= ret_expr - ret_expr_or_cond ::= ret_cond + ret_expr_or_cond ::= if_exp_ret stmt ::= return_lambda @@ -405,7 +405,7 @@ class Python37Parser(Python37BaseParser): def p_32on(self, args): """ - conditional::= expr jmp_false expr jump_forward_else expr COME_FROM + if_exp::= expr jmp_false expr jump_forward_else expr COME_FROM # compare_chained2 is used in a "chained_compare": x <= y <= z # used exclusively in compare_chained @@ -630,8 +630,8 @@ class Python37Parser(Python37BaseParser): def p_37conditionals(self, args): """ - expr ::= conditional37 - conditional37 ::= expr expr jf_cfs expr COME_FROM + expr ::= if_exp37 + if_exp37 ::= expr expr jf_cfs expr COME_FROM jf_cfs ::= JUMP_FORWARD _come_froms ifelsestmt ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except @@ -712,18 +712,18 @@ class Python37Parser(Python37BaseParser): def p_expr3(self, args): """ - expr ::= conditionalnot - conditionalnot ::= expr jmp_true expr jump_forward_else expr COME_FROM + expr ::= if_exp_not + if_exp_not ::= expr jmp_true expr jump_forward_else expr COME_FROM # a JUMP_FORWARD to another JUMP_FORWARD can get turned into # a JUMP_ABSOLUTE with no COME_FROM - conditional ::= expr jmp_false expr jump_absolute_else expr + if_exp ::= expr jmp_false expr jump_absolute_else expr - # if_expr_true are for conditions which always evaluate true + # if_exp_true are for conditions which always evaluate true # There is dead or non-optional remnants of the condition code though, # and we use that to match on to reconstruct the source more accurately - expr ::= if_expr_true - if_expr_true ::= expr JUMP_FORWARD expr COME_FROM + expr ::= if_exp_true + if_exp_true ::= expr JUMP_FORWARD expr COME_FROM """ def p_generator_exp3(self, args): @@ -920,9 +920,9 @@ class Python37Parser(Python37BaseParser): jmp_true ::= POP_JUMP_IF_TRUE # FIXME: Common with 2.7 - ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM - ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM - ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond + ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM + ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM + if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond jitop_come_from ::= JUMP_IF_TRUE_OR_POP come_froms jifop_come_from ::= JUMP_IF_FALSE_OR_POP come_froms @@ -966,26 +966,27 @@ class Python37Parser(Python37BaseParser): def p_stmt3(self, args): """ - stmt ::= if_expr_lambda - stmt ::= conditional_not_lambda + stmt ::= if_exp_lambda + stmt ::= if_exp_not_lambda # If statement inside a loop: stmt ::= ifstmtl - if_expr_lambda ::= expr jmp_false expr return_if_lambda + if_exp_lambda ::= expr jmp_false expr return_if_lambda return_stmt_lambda LAMBDA_MARKER - conditional_not_lambda + if_exp_not_lambda ::= expr jmp_true expr return_if_lambda return_stmt_lambda LAMBDA_MARKER return_stmt_lambda ::= ret_expr RETURN_VALUE_LAMBDA return_if_lambda ::= RETURN_END_IF_LAMBDA - stmt ::= return_closure - return_closure ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST + stmt ::= return_closure + return_closure ::= LOAD_CLOSURE RETURN_VALUE RETURN_LAST - stmt ::= whileTruestmt - ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms + stmt ::= whileTruestmt + ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite _come_froms + ifelsestmtl ::= testexpr c_stmts_opt jump_forward_else else_suitec ifstmtl ::= testexpr _ifstmts_jumpl @@ -1091,7 +1092,7 @@ class Python37Parser(Python37BaseParser): jf_cf ::= JUMP_FORWARD COME_FROM cf_jf_else ::= come_froms JUMP_FORWARD ELSE - conditional ::= expr jmp_false expr jf_cf expr COME_FROM + if_exp ::= expr jmp_false expr jf_cf expr COME_FROM async_for_stmt ::= setup_loop expr GET_AITER @@ -1207,25 +1208,31 @@ class Python37Parser(Python37BaseParser): elif opname == "BEFORE_ASYNC_WITH": rules_str = """ - stmt ::= async_with_stmt + stmt ::= async_with_stmt SETUP_ASYNC_WITH + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH + async_with_post ::= COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH END_FINALLY + + stmt ::= async_with_as_stmt async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store - suite_stmts_opt - POP_BLOCK LOAD_CONST - COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY - stmt ::= async_with_as_stmt - async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts_opt - POP_BLOCK LOAD_CONST - COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY + async_with_pre + store + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + async_with_post """ self.addRule(rules_str, nop_func) diff --git a/uncompyle6/parsers/parse37base.py b/uncompyle6/parsers/parse37base.py index ce287601..11893f4d 100644 --- a/uncompyle6/parsers/parse37base.py +++ b/uncompyle6/parsers/parse37base.py @@ -159,12 +159,11 @@ class Python37BaseParser(PythonParser): stmt ::= assign2_pypy assign3_pypy ::= expr expr expr store store store assign2_pypy ::= expr expr store store - stmt ::= if_expr_lambda - stmt ::= conditional_not_lambda - if_expr_lambda ::= expr jmp_false expr return_if_lambda + stmt ::= if_exp_lambda + stmt ::= if_exp_not_lambda + if_exp_lambda ::= expr jmp_false expr return_if_lambda return_lambda LAMBDA_MARKER - conditional_not_lambda - ::= expr jmp_true expr return_if_lambda + if_exp_not_lambda ::= expr jmp_true expr return_if_lambda return_lambda LAMBDA_MARKER """, nop_func, @@ -211,37 +210,57 @@ class Python37BaseParser(PythonParser): if self.version < 3.8: rules_str += """ - async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY - async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store suite_stmts_opt - POP_BLOCK LOAD_CONST COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM - WITH_CLEANUP_FINISH END_FINALLY + stmt ::= async_with_stmt SETUP_ASYNC_WITH + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts_opt + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store + suite_stmts_opt + POP_BLOCK LOAD_CONST + async_with_post """ else: rules_str += """ - async_with_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH POP_TOP suite_stmts - POP_TOP POP_BLOCK BEGIN_FINALLY COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM + async_with_pre ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH + async_with_post ::= BEGIN_FINALLY COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM WITH_CLEANUP_FINISH END_FINALLY - async_with_as_stmt ::= expr - BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM - SETUP_ASYNC_WITH store suite_stmts - POP_TOP POP_BLOCK BEGIN_FINALLY COME_FROM_ASYNC_WITH - WITH_CLEANUP_START - GET_AWAITABLE LOAD_CONST YIELD_FROM + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts + POP_TOP POP_BLOCK + async_with_post + async_with_stmt ::= expr + async_with_pre + POP_TOP + suite_stmts + POP_BLOCK + BEGIN_FINALLY + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM + WITH_CLEANUP_FINISH POP_FINALLY LOAD_CONST RETURN_VALUE + COME_FROM_ASYNC_WITH + WITH_CLEANUP_START GET_AWAITABLE LOAD_CONST YIELD_FROM WITH_CLEANUP_FINISH END_FINALLY + async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_TOP POP_BLOCK + async_with_post + async_with_as_stmt ::= expr + async_with_pre + store suite_stmts + POP_BLOCK async_with_post """ self.addRule(rules_str, nop_func) @@ -993,6 +1012,7 @@ class Python37BaseParser(PythonParser): "_ifstmts_jump": ifstmts_jump, "and": and_check, "ifelsestmt": ifelsestmt, + "ifelsestmtl": ifelsestmt, "iflaststmt": iflaststmt, "iflaststmtl": iflaststmt, "ifstmt": ifstmt, @@ -1013,6 +1033,7 @@ class Python37BaseParser(PythonParser): self.check_reduce["while1elsestmt"] = "noAST" self.check_reduce["_ifstmts_jump"] = "AST" self.check_reduce["ifelsestmt"] = "AST" + self.check_reduce["ifelsestmtl"] = "AST" self.check_reduce["iflaststmt"] = "AST" self.check_reduce["iflaststmtl"] = "AST" self.check_reduce["ifstmt"] = "AST" diff --git a/uncompyle6/parsers/reducecheck/ifelsestmt.py b/uncompyle6/parsers/reducecheck/ifelsestmt.py index 7583c2ea..319b9bd9 100644 --- a/uncompyle6/parsers/reducecheck/ifelsestmt.py +++ b/uncompyle6/parsers/reducecheck/ifelsestmt.py @@ -2,14 +2,9 @@ from uncompyle6.scanners.tok import Token - -def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): - if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP": - # ifelsestmt jumped outside of loop. No good. - return True - - if rule not in ( - ( +IFELSE_STMT_RULES = frozenset( + [ + ( "ifelsestmt", ( "testexpr", @@ -29,6 +24,24 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): "\\e__come_froms", ), ), + ( + "ifelsestmtl", + ( + "testexpr", + "c_stmts_opt", + "jump_forward_else", + "else_suitec", + ), + ), + ( + "ifelsestmtc", + ( + "testexpr", + "c_stmts_opt", + "jump_absolute_else", + "else_suitec" + ), + ), ( "ifelsestmt", ( @@ -72,84 +85,119 @@ def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): "else_suite", ), ), - ): + ]) + +def ifelsestmt(self, lhs, n, rule, ast, tokens, first, last): + + if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP" and lhs != "ifelsestmtc": + # ifelsestmt jumped outside of loop. No good. + return True + + if rule not in IFELSE_STMT_RULES: return False + + # Avoid if/else where the "then" is a "raise_stmt1" for an + # assert statemetn. Parse this as an "assert" instead. + stmts = ast[1] + if stmts in ("c_stmts",) and len(stmts) == 1: + raise_stmt1 = stmts[0] + if ( + raise_stmt1 == "raise_stmt1" and + raise_stmt1[0] in ("LOAD_ASSERT",) + ): + return True + # Make sure all of the "come froms" offset at the # end of the "if" come from somewhere inside the "if". # Since the come_froms are ordered so that lowest # offset COME_FROM is last, it is sufficient to test # just the last one. - come_froms = ast[-1] - if come_froms.kind != "else_suite" and self.version >= 3.0: - if come_froms == "opt_come_from_except" and len(come_froms) > 0: - come_froms = come_froms[0] - if not isinstance(come_froms, Token): - if len(come_froms): - return tokens[first].offset > come_froms[-1].attr - elif tokens[first].offset > come_froms.attr: - return True + if len(ast) == 5: + end_come_froms = ast[-1] + if end_come_froms.kind != "else_suite" and self.version >= 3.0: + if end_come_froms == "opt_come_from_except" and len(end_come_froms) > 0: + end_come_froms = end_come_froms[0] + if not isinstance(end_come_froms, Token): + if len(end_come_froms): + return tokens[first].offset > end_come_froms[-1].attr + elif tokens[first].offset > end_come_froms.attr: + return True - # FIXME: There is weirdness in the grammar we need to work around. - # we need to clean up the grammar. - if self.version < 3.0: - last_token = ast[-1] - else: - last_token = tokens[last] - if last_token == "COME_FROM" and tokens[first].offset > last_token.attr: - if self.version < 3.0 and self.insts[self.offset2inst_index[last_token.attr]].opname != "SETUP_LOOP": - return True + # FIXME: There is weirdness in the grammar we need to work around. + # we need to clean up the grammar. + if self.version < 3.0: + last_token = ast[-1] + else: + last_token = tokens[last] + if last_token == "COME_FROM" and tokens[first].offset > last_token.attr: + if self.version < 3.0 and self.insts[self.offset2inst_index[last_token.attr]].opname != "SETUP_LOOP": + return True testexpr = ast[0] # Check that the condition portion of the "if" # jumps to the "else" part. if testexpr[0] in ("testtrue", "testfalse"): - test = testexpr[0] + if_condition = testexpr[0] else_suite = ast[3] - assert else_suite == "else_suite" + assert else_suite.kind.startswith("else_suite") - if len(test) > 1 and test[1].kind.startswith("jmp_"): + if len(if_condition) > 1 and if_condition[1].kind.startswith("jmp_"): if last == n: last -= 1 - jmp = test[1] + jmp = if_condition[1] if self.version > 2.6: jmp_target = jmp[0].attr else: jmp_target = int(jmp[0].pattr) - # Make sure we don't jump at the end of the "then" inside the "else" - # (jf_cf_pop may be a 2.6ish specific thing.) - jump_forward = ast[2] - if jump_forward == "jf_cf_pop": - jump_forward = jump_forward[0] + # Below we check that jmp_target is jumping to a feasible + # location. It should be to the transition after the "then" + # block and to the beginning of the "else" block. + # However the "if/else" is inside a loop the false test can be + # back to the loop. + + # FIXME: the below logic for jf_cfs could probably be + # simplified. + jump_else_end = ast[2] + if jump_else_end == "jf_cf_pop": + jump_else_end = jump_else_end[0] jump_to_jump = False - if jump_forward == "JUMP_FORWARD": + if jump_else_end == "JUMP_FORWARD": jump_to_jump = True - endif_target = int(jump_forward.pattr) + endif_target = int(jump_else_end.pattr) last_offset = tokens[last].off2int() if endif_target != last_offset: return True + last_offset = tokens[last].off2int(prefer_last=False) + if jmp_target == last_offset: + # jmp_target should be jumping to the end of the if/then/else + # but is it jumping to the beginning of the "else" + return True + if ( + jump_else_end in ("jf_cfs", "jump_forward_else") + and jump_else_end[0] == "JUMP_FORWARD" + ): + # If the "else" jump jumps before the end of the the "if .. else end", then this + # is not this kind of "ifelsestmt". + jump_else_forward = jump_else_end[0] + jump_else_forward_target = jump_else_forward.attr + if jump_else_forward_target < last_offset: + return True + pass + if ( + jump_else_end in ("jb_elsec", "jb_elsel", "jf_cfs", "jb_cfs") + and jump_else_end[-1] == "COME_FROM" + ): + if jump_else_end[-1].off2int() != jmp_target: + return True - # The jump inside "else" check below should be added. - # add this until we can find out what's wrong with - # not being able to parse: - # if a and b or c: - # x = 1 - # else: - # x = 2 - - # FIXME: add this - # if jmp_target < else_suite.first_child().off2int(): - # return True - - if tokens[first].off2int() > jmp_target and not jump_to_jump: + if tokens[first].off2int() > jmp_target: return True - return (jmp_target > tokens[last].off2int()) and tokens[ - last - ] != "JUMP_FORWARD" + return (jmp_target > last_offset) and tokens[last] != "JUMP_FORWARD" return False diff --git a/uncompyle6/parsers/reducecheck/ifstmt.py b/uncompyle6/parsers/reducecheck/ifstmt.py index 2aefac45..d2abe490 100644 --- a/uncompyle6/parsers/reducecheck/ifstmt.py +++ b/uncompyle6/parsers/reducecheck/ifstmt.py @@ -7,7 +7,9 @@ def ifstmt(self, lhs, n, rule, ast, tokens, first, last): last -= 1 pass if tokens[last].attr and isinstance(tokens[last].attr, int): - return tokens[first].offset < tokens[last].attr + if tokens[first].offset >= tokens[last].attr: + return True + pass pass # Make sure jumps don't extend beyond the end of the if statement. diff --git a/uncompyle6/parsers/reducecheck/or_check.py b/uncompyle6/parsers/reducecheck/or_check.py index e72c33be..c9af5be8 100644 --- a/uncompyle6/parsers/reducecheck/or_check.py +++ b/uncompyle6/parsers/reducecheck/or_check.py @@ -1,9 +1,9 @@ # Copyright (c) 2020 Rocky Bernstein - +ASSERT_OPS = frozenset(["LOAD_ASSERT", "RAISE_VARARGS_1"]) def or_check(self, lhs, n, rule, ast, tokens, first, last): if rule == ("or", ("expr", "jmp_true", "expr")): - if tokens[last] in ("LOAD_ASSERT", "RAISE_VARARGS_1",): + if tokens[last] in ASSERT_OPS or tokens[last-1] in ASSERT_OPS: return True # The following test is be the most accurate. It prevents "or" from being diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index 10975e1f..fa43a70e 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -56,12 +56,12 @@ PRECEDENCE = { '_mklambda': 30, - 'conditional': 28, # Conditional expression - 'conditional_lamdba': 28, # Lambda expression - 'conditional_not_lamdba': 28, # Lambda expression - 'conditionalnot': 28, - 'if_expr_true': 28, - 'ret_cond': 28, + 'if_exp': 28, # If_Exp expression + 'if_exp_lamdba': 28, # Lambda expression + 'if_exp_not_lamdba': 28, # Lambda expression + 'if_exp_not': 28, + 'if_exp_true': 28, + 'if_exp_ret': 28, 'or': 26, # Boolean OR 'ret_or': 26, @@ -92,6 +92,8 @@ PRECEDENCE = { 'BINARY_POWER': 4, # Exponentiation, * + 'await_expr': 3, # await x, * + 'attribute': 2, # x.attribute 'buildslice2': 2, # x[index] 'buildslice3': 2, # x[index:index] @@ -301,24 +303,24 @@ TABLE_DIRECT = { # which we don't use here. 'aug_assign1': ( '%|%c %c %c\n', 0, 2, 1), - 'aug_assign2': ( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4 ), - 'designList': ( '%c = %c', 0, -1 ), + 'aug_assign2': ( '%|%c.%[2]{pattr} %c %c\n', 0, -3, -4 ), + 'designList': ( '%c = %c', 0, -1 ), 'and': ( '%c and %c', 0, 2 ), 'ret_and': ( '%c and %c', 0, 2 ), 'and2': ( '%c', 3 ), 'or': ( '%c or %c', 0, 2 ), 'ret_or': ( '%c or %c', 0, 2 ), - 'conditional': ( '%p if %c else %c', + 'if_exp': ( '%p if %c else %c', (2, 'expr', 27), 0, 4 ), - 'if_expr_lambda': ( '%p if %c else %c', + 'if_exp_lambda': ( '%p if %c else %c', (2, 'expr', 27), (0, 'expr'), 4 ), - 'if_expr_true': ( '%p if 1 else %c', (0, 'expr', 27), 2 ), - 'ret_cond': ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ), - 'conditional_not': ( '%p if not %p else %p', + 'if_exp_true': ( '%p if 1 else %c', (0, 'expr', 27), 2 ), + 'if_exp_ret': ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ), + 'if_exp_not': ( '%p if not %p else %p', (2, 27), (0, "expr", PRECEDENCE['unary_not']), (4, 27) ), - 'conditional_not_lambda': + 'if_exp_not_lambda': ( '%p if not %c else %c', (2, 'expr', 27), 0, 4 ), diff --git a/uncompyle6/semantics/customize3.py b/uncompyle6/semantics/customize3.py index ebd38adb..5e670763 100644 --- a/uncompyle6/semantics/customize3.py +++ b/uncompyle6/semantics/customize3.py @@ -36,7 +36,7 @@ def customize_for_version3(self, version): TABLE_DIRECT.update( { "comp_for": (" for %c in %c", (2, "store"), (0, "expr")), - "conditionalnot": ( + "if_exp_not": ( "%c if not %c else %c", (2, "expr"), (0, "expr"), diff --git a/uncompyle6/semantics/customize35.py b/uncompyle6/semantics/customize35.py index 587eadc5..23c3b6d2 100644 --- a/uncompyle6/semantics/customize35.py +++ b/uncompyle6/semantics/customize35.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019 by Rocky Bernstein +# Copyright (c) 2019-2020 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +17,12 @@ from xdis.code import iscode from xdis.util import COMPILER_FLAG_BIT -from uncompyle6.semantics.consts import INDENT_PER_LEVEL, TABLE_DIRECT +from uncompyle6.semantics.consts import ( + INDENT_PER_LEVEL, + PRECEDENCE, + TABLE_DIRECT, +) + from uncompyle6.semantics.helper import flatten_list, gen_function_parens_adjust ####################### @@ -26,7 +31,11 @@ from uncompyle6.semantics.helper import flatten_list, gen_function_parens_adjust def customize_for_version35(self, version): TABLE_DIRECT.update( { - "await_expr": ("await %c", 0), + # nested await expressions like: + # return await (await bar()) + # need parenthesis. + "await_expr": ("await %p", (0, PRECEDENCE["await_expr"]-1)), + "await_stmt": ("%|%c\n", 0), "async_for_stmt": ("%|async for %c in %c:\n%+%|%c%-\n\n", 9, 1, 25), "async_forelse_stmt": ( @@ -36,12 +45,12 @@ def customize_for_version35(self, version): 25, (27, "else_suite"), ), - "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 7), + "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 3), "async_with_as_stmt": ( "%|async with %c as %c:\n%+%c%-", (0, "expr"), - (6, "store"), - 7, + (2, "store"), + 3, ), "unmap_dict": ("{**%C}", (0, -1, ", **")), # "unmapexpr": ( "{**%c}", 0), # done by n_unmapexpr diff --git a/uncompyle6/semantics/customize36.py b/uncompyle6/semantics/customize36.py index d2c672be..944b28e3 100644 --- a/uncompyle6/semantics/customize36.py +++ b/uncompyle6/semantics/customize36.py @@ -49,29 +49,30 @@ def customize_for_version36(self, version): TABLE_DIRECT.update( { - "tryfinally36": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", (1, "returns"), 3), - "func_args36": ("%c(**", 0), - "try_except36": ("%|try:\n%+%c%-%c\n\n", 1, -2), - "except_return": ("%|except:\n%+%c%-", 3), - "unpack_list": ("*%c", (0, "list")), - "tryfinally_return_stmt": ("%|try:\n%+%c%-%|finally:\n%+%|return%-\n\n", 1), - "async_for_stmt36": ( - "%|async for %c in %c:\n%+%c%-%-\n\n", - (9, "store"), - (1, "expr"), - (18, "for_block"), - ), - "call_ex": ("%c(%p)", (0, "expr"), (1, 100)), - # This comes from 3.7. Eventually we will rebase from 3.7 - # and then this can go away - "conditional37": ("%p if %c else %c", (1, "expr", 27), 0, 3), - "store_annotation": ("%[1]{pattr}: %c", 0), "ann_assign_init_value": ( "%|%c = %p\n", (-1, "store_annotation"), (0, "expr", 200), ), "ann_assign_no_init": ("%|%c\n", (0, "store_annotation")), + "async_for_stmt36": ( + "%|async for %c in %c:\n%+%c%-\n\n", + (9, "store"), + (1, "expr"), + (18, "for_block"), + ), + "call_ex": ("%c(%p)", (0, "expr"), (1, 100)), + "except_return": ("%|except:\n%+%c%-", 3), + "func_args36": ("%c(**", 0), + # This comes from 3.7. Eventually we will rebase from 3.7 + # and then this can go away + "if_exp37": ("%p if %c else %c", (1, "expr", 27), 0, 3), + "ifstmtl": ("%|if %c:\n%+%c%-", (0, "testexpr"), (1, "_ifstmts_jumpl")), + "try_except36": ("%|try:\n%+%c%-%c\n\n", 1, -2), + "tryfinally36": ("%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", (1, "returns"), 3), + "tryfinally_return_stmt": ("%|try:\n%+%c%-%|finally:\n%+%|return%-\n\n", 1), + "unpack_list": ("*%c", (0, "list")), + "store_annotation": ("%[1]{pattr}: %c", 0), } ) diff --git a/uncompyle6/semantics/customize37.py b/uncompyle6/semantics/customize37.py index 09f96d8f..0bd5d934 100644 --- a/uncompyle6/semantics/customize37.py +++ b/uncompyle6/semantics/customize37.py @@ -55,24 +55,18 @@ def customize_for_version37(self, version): (1, "expr"), (17, "for_block"), ), - "async_for_stmt36": ( - "%|async for %c in %c:\n%+%c%-%-\n\n", - (9, "store"), - (1, "expr"), - (18, "for_block"), - ), "async_for_stmt37": ( - "%|async for %c in %c:\n%+%c%-%-\n\n", + "%|async for %c in %c:\n%+%c%-\n\n", (7, "store"), (1, "expr"), (16, "for_block"), ), - "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 7), + "async_with_stmt": ("%|async with %c:\n%+%c%-", (0, "expr"), 3), "async_with_as_stmt": ( "%|async with %c as %c:\n%+%c%-", (0, "expr"), - (6, "store"), - 7, + (2, "store"), + 3, ), "async_forelse_stmt": ( "%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", @@ -85,7 +79,12 @@ def customize_for_version37(self, version): "attributes37": ("%[0]{pattr} import %c", (0, "IMPORT_NAME_ATTR"), (1, "IMPORT_FROM")), - "await_expr": ("await %c", 0), + + # nested await expressions like: + # return await (await bar()) + # need parenthesis. + "await_expr": ("await %p", (0, PRECEDENCE["await_expr"]-1)), + "await_stmt": ("%|%c\n", 0), "call_ex": ("%c(%p)", (0, "expr"), (1, 100)), "compare_chained1a_37": ( @@ -121,8 +120,8 @@ def customize_for_version37(self, version): (0, 19), (6, 19), ), - 'conditional37': ( '%p if %c else %c', - (1, 'expr', 27), 0, 3 ), + 'if_exp37': ( '%p if %c else %c', + (1, 'expr', 27), 0, 3 ), "except_return": ("%|except:\n%+%c%-", 3), "if_exp_37a": ( diff --git a/uncompyle6/semantics/helper.py b/uncompyle6/semantics/helper.py index 1fbde649..cbb643fd 100644 --- a/uncompyle6/semantics/helper.py +++ b/uncompyle6/semantics/helper.py @@ -68,7 +68,7 @@ def find_all_globals(node, globs): # # print("XXX", n.kind, global_ops) # if isinstance(n, SyntaxTree): # # FIXME: do I need a caser for n.kind="mkfunc"? -# if n.kind in ("if_expr_lambda", "return_lambda"): +# if n.kind in ("if_exp_lambda", "return_lambda"): # globs = find_globals(n, globs, mklambda_globals) # else: # globs = find_globals(n, globs, global_ops) diff --git a/uncompyle6/semantics/make_function3.py b/uncompyle6/semantics/make_function3.py index 073c38f6..551d5aa8 100644 --- a/uncompyle6/semantics/make_function3.py +++ b/uncompyle6/semantics/make_function3.py @@ -676,7 +676,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None): # add in "yield" just so that the compiler will mark # the GENERATOR bit of the function. See for example # Python 3.x's test_generator.py test program. - if code.co_flags & CO_GENERATOR: + if not is_lambda and code.co_flags & CO_GENERATOR: need_bogus_yield = True for token in scanner_code._tokens: if token in ("YIELD_VALUE", "YIELD_FROM"): diff --git a/uncompyle6/semantics/make_function36.py b/uncompyle6/semantics/make_function36.py index c01e577e..27686573 100644 --- a/uncompyle6/semantics/make_function36.py +++ b/uncompyle6/semantics/make_function36.py @@ -355,7 +355,7 @@ def make_function36(self, node, is_lambda, nested=1, code_node=None): # add in "yield" just so that the compiler will mark # the GENERATOR bit of the function. See for example # Python 3.x's test_connection.py and test_contexlib_async test programs. - if code.co_flags & (CO_GENERATOR | CO_ASYNC_GENERATOR): + if not is_lambda and code.co_flags & (CO_GENERATOR | CO_ASYNC_GENERATOR): need_bogus_yield = True for token in scanner_code._tokens: if token == "YIELD_VALUE": diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 93e371a7..d85e68cc 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -2260,11 +2260,7 @@ class SourceWalker(GenericASTTraversal, object): def build_class(self, code): """Dump class definition, doc string and class body.""" - try: - assert iscode(code) - except: - from trepan.api import debug; debug() - + assert iscode(code) self.classes.append(self.currentclass) code = Code(code, self.scanner, self.currentclass) @@ -2274,11 +2270,14 @@ class SourceWalker(GenericASTTraversal, object): code._tokens = None # save memory assert ast == "stmts" + if ast[0] == "sstmt": + ast[0] = ast[0][0] + first_stmt = ast[0] + if ast[0] == "docstring": self.println(self.traverse(ast[0])) del ast[0] - first_stmt = ast[0] if 3.0 <= self.version <= 3.3: try: if first_stmt == "store_locals": @@ -2298,6 +2297,11 @@ class SourceWalker(GenericASTTraversal, object): pass have_qualname = False + if len(ast): + if ast[0] == "sstmt": + ast[0] = ast[0][0] + first_stmt = ast[0] + if self.version < 3.0: # Should we ditch this in favor of the "else" case? qualname = ".".join(self.classes) @@ -2311,7 +2315,7 @@ class SourceWalker(GenericASTTraversal, object): ], ) # FIXME: is this right now that we've redone the grammar? - have_qualname = ast[0][0] == QUAL_NAME + have_qualname = ast[0] == QUAL_NAME else: # Python 3.4+ has constants like 'cmp_to_key..K' # which are not simple classes like the < 3 case. @@ -2342,6 +2346,7 @@ class SourceWalker(GenericASTTraversal, object): do_doc = True if do_doc and self.hide_internal: try: + # FIXME: Is there an extra [0]? docstring = ast[i][0][0][0][0].pattr except: docstring = code.co_consts[0] @@ -2349,18 +2354,31 @@ class SourceWalker(GenericASTTraversal, object): self.println() del ast[i] - # the function defining a class normally returns locals(); we - # don't want this to show up in the source, thus remove the node - if len(ast) > 0 and ast[-1][0] == RETURN_LOCALS: - if self.hide_internal: - del ast[-1] # remove last node - # else: - # print ast[-1][-1] + # The function defining a class returns locals() in Python somewhere less than + # 3.7. + # + # We don't want this to show up in the source, so remove the node. + if len(ast): + if ast == "stmts" and ast[-1] == "sstmt": + return_locals_parent = ast[-1] + parent_index = 0 + else: + return_locals_parent = ast + parent_index = -1 + return_locals = return_locals_parent[parent_index] + if return_locals == RETURN_LOCALS: + if self.hide_internal: + del return_locals_parent[parent_index] + pass + pass + # else: + # print stmt[-1] + + # Add "global" declaration statements at the top globals, nonlocals = find_globals_and_nonlocals( ast, set(), set(), code, self.version ) - # Add "global" declaration statements at the top # of the function for g in sorted(globals): self.println(indent, "global ", g) @@ -2371,8 +2389,11 @@ class SourceWalker(GenericASTTraversal, object): old_name = self.name self.gen_source(ast, code.co_name, code._customize) self.name = old_name + + # save memory by deleting no-longer-used structures code._tokens = None - code._customize = None # save memory + code._customize = None + self.classes.pop(-1) def gen_source(self, ast, name, customize, is_lambda=False, returnNone=False): diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index 850149c6..f8bbd041 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -121,14 +121,14 @@ class TreeTransform(GenericASTTraversal, object): if ifstmts_jump == "_ifstmts_jumpl" and ifstmts_jump[0] == "_ifstmts_jump": ifstmts_jump = ifstmts_jump[0] - elif ifstmts_jump not in ("_ifstmts_jump", "ifstmts_jumpl"): + elif ifstmts_jump not in ("_ifstmts_jump", "_ifstmts_jumpl", "ifstmts_jumpl"): return node stmts = ifstmts_jump[0] else: # iflaststmtl works this way stmts = node[1] - if stmts in ("c_stmts", "stmts") and len(stmts) == 1: + if stmts in ("c_stmts", "stmts", "stmts_opt") and len(stmts) == 1: raise_stmt = stmts[0] if raise_stmt != "raise_stmt1": raise_stmt = raise_stmt[0] @@ -139,13 +139,16 @@ class TreeTransform(GenericASTTraversal, object): and 1 <= len(testtrue_or_false) <= 2 and raise_stmt.first_child().pattr == "AssertionError" ): - if testtrue_or_false == "testtrue": + if testtrue_or_false in ("testtrue", "testtruel"): # Skip over the testtrue because because it would # produce a "not" and we don't want that here. assert_expr = testtrue_or_false[0] jump_cond = NoneToken else: - assert testtrue_or_false == "testfalse" + try: + assert testtrue_or_false in ("testfalse", "testfalsel") + except: + from trepan.api import debug; debug() assert_expr = testtrue_or_false[0] if assert_expr == "testfalse_not_and": # FIXME: come pack to stuff like this