From 223804ac1f6a3b5b0aa301a6179376e98a2505cf Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 14 May 2022 09:51:50 -0400 Subject: [PATCH 01/10] semi-black scanner26.py --- uncompyle6/scanners/scanner26.py | 68 ++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index a1067fbd..f0d46f90 100755 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -26,6 +26,7 @@ import sys import uncompyle6.scanners.scanner2 as scan # bytecode verification, verify(), uses JUMP_OPs from here +from xdis import iscode from xdis.opcodes import opcode_26 from xdis.bytecode import _get_const_info @@ -72,7 +73,7 @@ class Scanner26(scan.Scanner2): bytecode = self.build_instructions(co) # show_asm = 'after' - if show_asm in ('both', 'before'): + if show_asm in ("both", "before"): for instr in bytecode.get_instructions(co): print(instr.disassemble()) @@ -81,7 +82,7 @@ class Scanner26(scan.Scanner2): customize = {} if self.is_pypy: - customize['PyPy'] = 1 + customize["PyPy"] = 0 codelen = len(self.code) @@ -93,6 +94,7 @@ class Scanner26(scan.Scanner2): # 'LOAD_ASSERT' is used in assert statements. self.load_asserts = set() for i in self.op_range(0, codelen): + # We need to detect the difference between: # raise AssertionError # and @@ -115,9 +117,9 @@ class Scanner26(scan.Scanner2): # Distinguish "print ..." from "print ...," if self.code[last_stmt] == self.opc.PRINT_ITEM: if self.code[i] == self.opc.PRINT_ITEM: - replace[i] = 'PRINT_ITEM_CONT' + replace[i] = "PRINT_ITEM_CONT" elif self.code[i] == self.opc.PRINT_NEWLINE: - replace[i] = 'PRINT_NEWLINE_CONT' + replace[i] = "PRINT_NEWLINE_CONT" last_stmt = i i = self.next_stmt[i] @@ -181,29 +183,25 @@ class Scanner26(scan.Scanner2): if op in self.opc.CONST_OPS: const = co.co_consts[oparg] - # We can't use inspect.iscode() because we may be - # using a different version of Python than the - # one that this was byte-compiled on. So the code - # types may mismatch. - if hasattr(const, 'co_name'): + if iscode(const): oparg = const - if const.co_name == '': - assert op_name == 'LOAD_CONST' - op_name = 'LOAD_LAMBDA' + if const.co_name == "": + assert op_name == "LOAD_CONST" + op_name = "LOAD_LAMBDA" elif const.co_name == self.genexpr_name: - op_name = 'LOAD_GENEXPR' - elif const.co_name == '': - op_name = 'LOAD_DICTCOMP' - elif const.co_name == '': - op_name = 'LOAD_SETCOMP' + op_name = "LOAD_GENEXPR" + elif const.co_name == "": + op_name = "LOAD_DICTCOMP" + elif const.co_name == "": + op_name = "LOAD_SETCOMP" else: op_name = "LOAD_CODE" - # verify uses 'pattr' for comparison, since 'attr' + # verify() uses 'pattr' for comparison, since 'attr' # now holds Code(const) and thus can not be used # for comparison (todo: think about changing this) - # pattr = 'code_object @ 0x%x %s->%s' % \ + # pattr = 'code_object @ 0x%x %s->%s' %\ # (id(const), const.co_filename, const.co_name) - pattr = '' + pattr = "" else: if oparg < len(co.co_consts): argval, _ = _get_const_info(oparg, co.co_consts) @@ -235,6 +233,7 @@ class Scanner26(scan.Scanner2): pattr = self.opc.cmp_op[oparg] elif op in self.opc.FREE_OPS: pattr = free[oparg] + if op in self.varargs_ops: # CE - Hack for >= 2.5 # Now all values loaded via LOAD_CLOSURE are packed into @@ -285,25 +284,36 @@ class Scanner26(scan.Scanner2): elif op == self.opc.LOAD_GLOBAL: if offset in self.load_asserts: - op_name = 'LOAD_ASSERT' + op_name = "LOAD_ASSERT" elif op == self.opc.RETURN_VALUE: if offset in self.return_end_ifs: - op_name = 'RETURN_END_IF' + op_name = "RETURN_END_IF" linestart = self.linestarts.get(offset, None) if offset not in replace: - tokens.append(Token( - op_name, oparg, pattr, offset, linestart, op, - has_arg, self.opc)) + tokens.append( + Token( + op_name, oparg, pattr, offset, linestart, op, has_arg, self.opc + ) + ) else: - tokens.append(Token( - replace[offset], oparg, pattr, offset, linestart, op, - has_arg, self.opc)) + tokens.append( + Token( + replace[offset], + oparg, + pattr, + offset, + linestart, + op, + has_arg, + self.opc, + ) + ) pass pass - if show_asm in ('both', 'after'): + if show_asm in ("both", "after"): for t in tokens: print(t.format(line_prefix="")) print() From 021810bb2cf9a9d3cc843e262b28e45a380c6b56 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 14 May 2022 16:54:50 -0400 Subject: [PATCH 02/10] Correct 2.x formatting "slice2" nonterminal --- uncompyle6/semantics/consts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index d711b7da..d094febf 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -256,7 +256,7 @@ TABLE_DIRECT = { (1, NO_PARENTHESIS_EVER) ), - "slice2": ( "[%c:%p]", + "slice2": ( "%c[:%p]", (0, "expr"), (1, NO_PARENTHESIS_EVER) ), From 656a9aa290d30e1c7fae146fbfd80e2d5ba02e4b Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 14 May 2022 17:42:30 -0400 Subject: [PATCH 03/10] Bugs in 2.x relative import '.' and 1.x bytecode --- uncompyle6/scanner.py | 2 ++ uncompyle6/scanners/scanner2.py | 6 ++++-- uncompyle6/scanners/scanner26.py | 2 +- uncompyle6/semantics/customize26_27.py | 15 +++++++++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 344c8ea1..d4fcfc4b 100644 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -226,6 +226,8 @@ class Scanner(object): # Locally we use list for more convenient iteration using indices linestarts = list(self.opc.findlinestarts(code_obj)) self.linestarts = dict(linestarts) + if not self.linestarts: + return [] # 'List-map' which shows line number of current op and offset of # first op on following line, given offset of op as index diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index db213bab..3b4b77da 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -521,7 +521,7 @@ class Scanner2(Scanner): for s in stmt_list: if code[s] == self.opc.JUMP_ABSOLUTE and s not in pass_stmts: target = self.get_target(s) - if target > s or self.lines[last_stmt].l_no == self.lines[s].l_no: + if target > s or (self.lines and self.lines[last_stmt].l_no == self.lines[s].l_no): stmts.remove(s) continue j = self.prev[s] @@ -622,6 +622,7 @@ class Scanner2(Scanner): parent = self.structs[0] start = parent["start"] end = parent["end"] + next_line_byte = end # Pick inner-most parent for our offset for struct in self.structs: @@ -650,7 +651,8 @@ class Scanner2(Scanner): if setup_target != loop_end_offset: self.fixed_jumps[offset] = loop_end_offset - (line_no, next_line_byte) = self.lines[offset] + if self.lines: + (line_no, next_line_byte) = self.lines[offset] # jump_back_offset is the instruction after the SETUP_LOOP # where we iterate back to. diff --git a/uncompyle6/scanners/scanner26.py b/uncompyle6/scanners/scanner26.py index f0d46f90..9c77476c 100755 --- a/uncompyle6/scanners/scanner26.py +++ b/uncompyle6/scanners/scanner26.py @@ -113,7 +113,7 @@ class Scanner26(scan.Scanner2): i = self.next_stmt[last_stmt] replace = {} while i < codelen - 1: - if self.lines[last_stmt].next > i: + if self.lines and self.lines[last_stmt].next > i: # Distinguish "print ..." from "print ...," if self.code[last_stmt] == self.opc.PRINT_ITEM: if self.code[i] == self.opc.PRINT_ITEM: diff --git a/uncompyle6/semantics/customize26_27.py b/uncompyle6/semantics/customize26_27.py index ee50bb21..2e503c21 100644 --- a/uncompyle6/semantics/customize26_27.py +++ b/uncompyle6/semantics/customize26_27.py @@ -30,10 +30,10 @@ def customize_for_version26_27(self, version): ######################################## if version > (2, 6): TABLE_DIRECT.update({ - 'except_cond2': ( '%|except %c as %c:\n', 1, 5 ), + "except_cond2": ( "%|except %c as %c:\n", 1, 5 ), # When a generator is a single parameter of a function, # it doesn't need the surrounding parenethesis. - 'call_generator': ('%c%P', 0, (1, -1, ', ', 100)), + "call_generator": ('%c%P', 0, (1, -1, ', ', 100)), }) else: TABLE_DIRECT.update({ @@ -60,3 +60,14 @@ def customize_for_version26_27(self, version): self.default(node) self.n_call = n_call + + def n_import_from(node): + import_name = node[2] + if import_name == "IMPORT_NAME" and import_name.pattr == "": + fmt = "%|from . import %c\n" + self.template_engine( + (fmt, (3, "importlist")), node + ) + self.prune() + self.default(node) + self.n_import_from = n_import_from From fa9cc4c66951e94f3b03dc9f25120811a1e478ef Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 14 May 2022 19:28:23 -0400 Subject: [PATCH 04/10] Correct 2.5-7 relative import formatting --- test/bytecode_2.5/01_rel_import.pyc | Bin 0 -> 254 bytes test/bytecode_2.6/01_rel_import.pyc | Bin 0 -> 254 bytes test/bytecode_2.7/01_rel_import.pyc | Bin 169 -> 254 bytes test/simple_source/stmts/01_rel_import.py | 2 ++ uncompyle6/semantics/consts.py | 13 +++++++++++-- uncompyle6/semantics/customize25.py | 6 ++++++ uncompyle6/semantics/customize26_27.py | 9 ++------- 7 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 test/bytecode_2.5/01_rel_import.pyc create mode 100644 test/bytecode_2.6/01_rel_import.pyc diff --git a/test/bytecode_2.5/01_rel_import.pyc b/test/bytecode_2.5/01_rel_import.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eff91378110247832659e5d8b683fe5aa1fe2cfe GIT binary patch literal 254 zcmX|(!485j42HWgj4?6cAvkanO+0uo#wYNAH!eijG=!0nbsY5MJNaflf!#1H>(~D6 z-+sR9ReE^W4g6nB`AmU(f~3!&1JDdg22660QP3PpK4}461X@5TNXKx^7PQzhLgwN8 z!P-1VZ28=asJa_78N8@(uS@2gjsl8+usscJ5q5n$n}I2n*!F`~6`IjDDo58l%(uJB bDqYdUShpFTxFqATH*(bK(~D6 z-+sR9ReE^W4g6nB`AmU(f~3!&1JDdg22660QP3PpK4}461X@5TNXKx^7PQzhLgwN8 z!P-1VZ28=asJa_78N8@(uS@2gjsl8+usscJ5q5n$n}I2n*!F`~6`IjDDo58l%(uJB bDqYdUShpFTZb`;tZ{(=e$sa;NNLa#t4K^v3 literal 0 HcmV?d00001 diff --git a/test/bytecode_2.7/01_rel_import.pyc b/test/bytecode_2.7/01_rel_import.pyc index 46c146daab02d1d9fb5d434f56862be67c236fc9..8e516b37d9f7e119bca0fb21d232b8d1c44e3ec5 100644 GIT binary patch literal 254 zcmYL>O$x#=5QQf{6%m9UqKmAof(sWSp1?)jbyJ!)N+BU7Nf)~E&R)PtDjG6h=1txk z_Fk`w!%KGXJrv(Fmfs_KyaY6WAy5+_J%yenO`)c<=AaoE4mC%e!Bs3UI1q%02pK}! z`{akXdY&=tZr1*VY@ZhqodFA~gp`g>*}5ua<5>LQ3vB&hMy(oeC)cUc2NQg`-8D|> a27jyzoAD8rXgnI*Pgmyscgk%2LVi6K~n87Ps- h2qZK>L 0: + node[2].pattr = ("." * node[0].pattr) + node[2].pattr + self.default(node) + self.n_import_from = n_import_from diff --git a/uncompyle6/semantics/customize26_27.py b/uncompyle6/semantics/customize26_27.py index 2e503c21..19196614 100644 --- a/uncompyle6/semantics/customize26_27.py +++ b/uncompyle6/semantics/customize26_27.py @@ -62,12 +62,7 @@ def customize_for_version26_27(self, version): self.n_call = n_call def n_import_from(node): - import_name = node[2] - if import_name == "IMPORT_NAME" and import_name.pattr == "": - fmt = "%|from . import %c\n" - self.template_engine( - (fmt, (3, "importlist")), node - ) - self.prune() + if node[0].pattr > 0: + node[2].pattr = ("." * node[0].pattr) + node[2].pattr self.default(node) self.n_import_from = n_import_from From 5c29b9a5e5da5eb8f5b361c469a1962b8b3ecd31 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 14 May 2022 20:28:24 -0400 Subject: [PATCH 05/10] remove \n in lambda; 2.6 grammar cleanup --- test/stdlib/2.4-exclude.sh | 2 +- test/stdlib/2.5-exclude.sh | 2 +- test/stdlib/2.7-exclude.sh | 1 - uncompyle6/parsers/parse26.py | 5 ----- uncompyle6/semantics/pysource.py | 7 ++----- 5 files changed, 4 insertions(+), 13 deletions(-) diff --git a/test/stdlib/2.4-exclude.sh b/test/stdlib/2.4-exclude.sh index a28d2cc6..664f6da5 100644 --- a/test/stdlib/2.4-exclude.sh +++ b/test/stdlib/2.4-exclude.sh @@ -42,7 +42,7 @@ SKIP_TESTS=( [test_decimal.py]=1 # [test_dis.py]=1 # We change line numbers - duh! [test_generators.py]=1 # Investigate - [test_grammar.py]=1 # Too many stmts. Handle large stmts + # [test_grammar.py]=1 # fails on its own - no module tests.test_support [test_grp.py]=1 # Long test - might work Control flow? [test_pep247.py]=1 # Long test - might work? Control flow? [test_socketserver.py]=1 # -- test takes too long to run: 40 seconds diff --git a/test/stdlib/2.5-exclude.sh b/test/stdlib/2.5-exclude.sh index 783e50c7..1b0d1afa 100644 --- a/test/stdlib/2.5-exclude.sh +++ b/test/stdlib/2.5-exclude.sh @@ -44,7 +44,7 @@ SKIP_TESTS=( [test_dis.py]=1 # We change line numbers - duh! [test_file.py]=1 # test assertion failures [test_generators.py]=1 # Investigate - [test_grammar.py]=1 # Too many stmts. Handle large stmts + # [test_grammar.py]=1 # fails on its own - no module tests.test_support [test_grp.py]=1 # Long test - might work Control flow? [test_macfs.py]=1 # it fails on its own [test_macostools.py]=1 # it fails on its own diff --git a/test/stdlib/2.7-exclude.sh b/test/stdlib/2.7-exclude.sh index 296cd679..629286a1 100644 --- a/test/stdlib/2.7-exclude.sh +++ b/test/stdlib/2.7-exclude.sh @@ -9,7 +9,6 @@ SKIP_TESTS=( [test_doctest.py]=1 # Fails on its own [test_exceptions.py]=1 [test_format.py]=1 # Control flow "and" vs nested "if" - [test_grammar.py]=1 # Too many stmts. Handle large stmts [test_grp.py]=1 # test takes to long, works interactively though [test_io.py]=1 # Test takes too long to run [test_ioctl.py]=1 # Test takes too long to run diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index d353c571..258d5f33 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -312,11 +312,6 @@ class Python26Parser(Python2Parser): compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false_then compare_chained2 _come_froms - return_lambda ::= RETURN_VALUE - return_lambda ::= RETURN_END_IF - return_lambda ::= RETURN_END_IF_LAMBDA - return_lambda ::= RETURN_VALUE_LAMBDA - compare_chained2 ::= expr COMPARE_OP return_expr_lambda compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA compare_chained2 ::= expr COMPARE_OP RETURN_END_IF COME_FROM diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 4ded56c2..6b1d495d 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -1167,11 +1167,8 @@ class SourceWalker(GenericASTTraversal, NonterminalActions, ComprehensionMixin): # f'{(lambda x:x)("8")!r}' # Adding a "\n" after "lambda x: x" will give an error message: # SyntaxError: f-string expression part cannot include a backslash - # So avoid that. - printfn = ( - self.write if self.in_format_string and is_lambda else self.println - ) - printfn(self.text) + # So avoid \n after writing text + self.write(self.text) self.name = old_name self.return_none = rn From 3f4e85695eaa3c9cdb233b6d53dd7264d9adc0ad Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 14 May 2022 22:24:16 -0400 Subject: [PATCH 06/10] Reduce 3.7 exclusion tests --- test/stdlib/3.7-exclude.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/stdlib/3.7-exclude.sh b/test/stdlib/3.7-exclude.sh index f17247b2..03995fe3 100644 --- a/test/stdlib/3.7-exclude.sh +++ b/test/stdlib/3.7-exclude.sh @@ -14,22 +14,19 @@ SKIP_TESTS=( # complicated control flow and "and/or" expressions [test_pickle.py]=1 - [test_builtin.py]=1 # FIXME works on decompyle6 [test_context.py]=1 # FIXME works on decompyle6 - [test_doctest2.py]=1 # FIXME works on decompyle6 [test_format.py]=1 # FIXME works on decompyle6 [test_marshal.py]=1 # FIXME works on decompyle6 [test_normalization.py]=1 # FIXME works on decompyle6 [test_os.py]=1 # FIXME works on decompyle6 - [test_pow.py]=1 # FIXME works on decompyle6 [test_slice.py]=1 # FIXME works on decompyle6 [test_sort.py]=1 # FIXME works on decompyle6 [test_statistics.py]=1 # FIXME works on decompyle6 [test_timeit.py]=1 # FIXME works on decompyle6 [test_urllib2_localnet.py]=1 # FIXME works on decompyle6 [test_urllib2.py]=1 # FIXME: works on uncompyle6 - [test_generators.py]=1 # FIXME: works on uncompyle6 - lambda parsing probably - [test_grammar.py]=1 # FIXME: works on uncompyle6 - lambda parsing probably + [test_generators.py]=1 # File "test_generators.py", line 44, in test_raise_and_yield_from self.assertEqual(exc.value, 'PASSED') + [test_grammar.py]=1 # FIXME: invalid syntax: l4 = lambda x=lambda y=lambda z=1: z: y(): x() [test___all__.py]=1 # it fails on its own [test_argparse.py]=1 #- it fails on its own From 8f7f0be7fafc7117a00b580c0b837bfa8f91f58b Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 20 May 2022 06:44:20 -0400 Subject: [PATCH 07/10] Administrivia Workflows CI: go back to released versions rather than github versions pyenv-newest-versions: updaed to use newest Python releases pypy38.py: fix wrong package name import 3.6-exclude.sh: update and advance --- .github/workflows/osx.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/windows.yml | 2 +- admin-tools/pyenv-newest-versions | 2 +- test/stdlib/3.7-exclude.sh | 1 - uncompyle6/scanners/pypy38.py | 4 ++-- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index c08a884b..16671d66 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -23,7 +23,7 @@ jobs: run: | python -m pip install --upgrade pip # Until the next xdis release - pip install git+https://github.com/rocky/python-xdis#egg=xdis + # pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install -e . pip install -r requirements-dev.txt - name: Test uncompyle6 diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 0294b57b..3123be7f 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -22,7 +22,7 @@ jobs: run: | python -m pip install --upgrade pip # Until the next xdis release - pip install git+https://github.com/rocky/python-xdis#egg=xdis + # pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install -e . pip install -r requirements-dev.txt - name: Test uncompyle6 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 91084679..f096f927 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -23,7 +23,7 @@ jobs: run: | python -m pip install --upgrade pip # Until the next xdis release - pip install git+https://github.com/rocky/python-xdis#egg=xdis + # pip install git+https://github.com/rocky/python-xdis#egg=xdis pip install -e . pip install -r requirements-dev.txt - name: Test uncompyle6 diff --git a/admin-tools/pyenv-newest-versions b/admin-tools/pyenv-newest-versions index 509daf6f..b6dd3491 100644 --- a/admin-tools/pyenv-newest-versions +++ b/admin-tools/pyenv-newest-versions @@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then echo "This script should be *sourced* rather than run directly through bash" exit 1 fi -export PYVERSIONS='3.6.15 3.7.13 pypy3.6-7.3.0 pyston-2.3.2 3.8.13 3.9.12 3.10.4' +export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.13 pypy3.8-7.3.9 pyston-2.3.3 3.8.13' diff --git a/test/stdlib/3.7-exclude.sh b/test/stdlib/3.7-exclude.sh index 03995fe3..32d2d342 100644 --- a/test/stdlib/3.7-exclude.sh +++ b/test/stdlib/3.7-exclude.sh @@ -127,7 +127,6 @@ SKIP_TESTS=( [test_traceback.py]=1 # Probably uses comment for testing [test_tracemalloc.py]=1 # test assert failres [test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds - [test_ttk_guionly.py]=1 # implementation specfic and test takes too long to run: 19 seconds [test_typing.py]=1 # parse error [test_types.py]=1 # parse error diff --git a/uncompyle6/scanners/pypy38.py b/uncompyle6/scanners/pypy38.py index e60ebeb1..f32c1bbd 100644 --- a/uncompyle6/scanners/pypy38.py +++ b/uncompyle6/scanners/pypy38.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 by Rocky Bernstein +# Copyright (c) 2021-2022 by Rocky Bernstein """ Python PyPy 3.8 decompiler scanner. @@ -6,7 +6,7 @@ Does some additional massaging of xdis-disassembled instructions to make things easier for decompilation. """ -import decompyle3.scanners.scanner38 as scan +import uncompyle6.scanners.scanner38 as scan # bytecode verification, verify(), uses JUMP_OPS from here from xdis.opcodes import opcode_38pypy as opc From 2cc58fec97311ae8dc8ba1b13b6178881adc5a66 Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 21 May 2022 12:30:21 -0400 Subject: [PATCH 08/10] Note narrowing bug --- .github/ISSUE_TEMPLATE/bug-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index cec076b4..2f9931e9 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -54,7 +54,7 @@ Bug reports that violate the above may be discarded. ## Description - + ## How to Reproduce From c08ab41afddc9eb5639c6df506ccf3b511e7098c Mon Sep 17 00:00:00 2001 From: rocky Date: Sat, 28 May 2022 15:45:51 -0400 Subject: [PATCH 09/10] Go over what constitutes a bug, yet again --- .github/ISSUE_TEMPLATE/bug-report.md | 11 +++++++++++ admin-tools/pyenv-versions | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index cec076b4..a7d72e66 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -11,6 +11,17 @@ Bugs are not for asking questions about a problem you are trying to solve that involve the use of uncompyle6 along the way, although I may be more tolerent of this if you sponsor the project. +Bugs are also not for general or novice kind help on how to install +this Python program in your environment in the way you would like to +have it set up, or how to interpret a Python traceback e.g. that winds +up saying Python X.Y.Z is not supported. + +For these kinds of things, you will save yourself time by asking +instead on forums like StackOverflow that are geared to helping people +for such general or novice kinds questions and tasks. And unless you +are a sponsor of the project, if your question seems to be of this +category, the issue may just be closed. + Also, the unless you are a sponsor of the project, it may take a while, maybe a week or so, before the bug report is noticed, let alone acted upon. diff --git a/admin-tools/pyenv-versions b/admin-tools/pyenv-versions index 9ce70332..7c1bcafa 100644 --- a/admin-tools/pyenv-versions +++ b/admin-tools/pyenv-versions @@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then echo "This script should be *sourced* rather than run directly through bash" exit 1 fi -export PYVERSIONS='3.7.11 3.8.12 3.9.7 3.10.0' +export PYVERSIONS='3.7.13 pyston-2.3.3 3.8.13' From 7fb483c5666ae498a8dc5a1dd051fa816ae03239 Mon Sep 17 00:00:00 2001 From: rocky Date: Wed, 8 Jun 2022 12:04:12 -0400 Subject: [PATCH 10/10] Sync up with decompile3's 3.8 try/else handling --- test/bytecode_3.8/02_tryfinally_return.pyc | Bin 242 -> 256 bytes uncompyle6/parsers/parse38.py | 90 +++-- uncompyle6/semantics/customize38.py | 376 ++++++++++++++------- 3 files changed, 325 insertions(+), 141 deletions(-) diff --git a/test/bytecode_3.8/02_tryfinally_return.pyc b/test/bytecode_3.8/02_tryfinally_return.pyc index 9b946df39ec492ae4274a8ca5facfba418c13ed8..c46b7c6eaea34e4b9b1ba29cdcca5386b367a546 100644 GIT binary patch delta 65 zcmeyw*ucaXI+0OkqNIin0}wC*F*^`D197ngkVs*u;z(hvWe8_5WmpW9PGJaUh+trx Mm>kZ?GI6\R@DF6Tf delta 55 zcmZo*`ozfSIgwFjqN0itBLf2i8xT7Kaj_7PsNzUrtYrviFlAWGkirnm5HYbZoRMkb GNpk=XwhF`m diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index e29accbf..d38abced 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2020 Rocky Bernstein +# Copyright (c) 2017-2020, 2022 Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,34 +22,31 @@ from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG from uncompyle6.parsers.parse37 import Python37Parser class Python38Parser(Python37Parser): - def p_38walrus(self, args): - """ - # named_expr is also known as the "walrus op" := - expr ::= named_expr - named_expr ::= expr DUP_TOP store - """ - - def p_38misc(self, args): + def p_38_stmt(self, args): """ stmt ::= async_for_stmt38 stmt ::= async_forelse_stmt38 + stmt ::= call_stmt + stmt ::= continue stmt ::= for38 - stmt ::= forelsestmt38 stmt ::= forelselaststmt38 stmt ::= forelselaststmtl38 - stmt ::= tryfinally38stmt + stmt ::= forelsestmt38 + stmt ::= try_elsestmtl38 + stmt ::= try_except38 + stmt ::= try_except38r + stmt ::= try_except38r2 + stmt ::= try_except38r3 + stmt ::= try_except38r4 + stmt ::= try_except_as + stmt ::= try_except_ret38 + stmt ::= tryfinally38astmt stmt ::= tryfinally38rstmt stmt ::= tryfinally38rstmt2 stmt ::= tryfinally38rstmt3 - stmt ::= tryfinally38astmt - stmt ::= try_elsestmtl38 - stmt ::= try_except_ret38 - stmt ::= try_except38 - stmt ::= try_except_as - stmt ::= whilestmt38 + stmt ::= tryfinally38stmt stmt ::= whileTruestmt38 - stmt ::= call_stmt - stmt ::= continue + stmt ::= whilestmt38 call_stmt ::= call break ::= POP_BLOCK BREAK_LOOP @@ -129,6 +126,10 @@ class Python38Parser(Python37Parser): forelselaststmt38 ::= expr get_for_iter store for_block POP_BLOCK else_suitec forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel + returns_in_except ::= _stmts except_return_value + except_return_value ::= POP_BLOCK return + except_return_value ::= expr POP_BLOCK RETURN_VALUE + whilestmt38 ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK whilestmt38 ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms @@ -155,9 +156,53 @@ class Python38Parser(Python37Parser): else_suitel opt_come_from_except try_except ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK except_handler38 + try_except38 ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt except_handler38a + # suite_stmts has a return + try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts + except_handler38b + try_except38r ::= SETUP_FINALLY return_except + except_handler38b + return_except ::= stmts POP_BLOCK return + + + # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there + # is a final return in the "except" block. + # So we treat the "return" separate from the other statements + cond_except_stmt ::= except_cond1 except_stmts + cond_except_stmts_opt ::= cond_except_stmt* + + try_except38r2 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP + cond_except_stmts_opt + POP_EXCEPT return + END_FINALLY + COME_FROM + + try_except38r3 ::= SETUP_FINALLY + suite_stmts_opt + POP_BLOCK JUMP_FORWARD + COME_FROM_FINALLY + cond_except_stmts_opt + POP_EXCEPT return + COME_FROM + END_FINALLY + COME_FROM + + + try_except38r4 ::= SETUP_FINALLY + returns_in_except + COME_FROM_FINALLY + except_cond1 + return + COME_FROM + END_FINALLY + + # suite_stmts has a return try_except38 ::= SETUP_FINALLY POP_BLOCK suite_stmts except_handler38b @@ -234,6 +279,13 @@ class Python38Parser(Python37Parser): POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP """ + def p_38walrus(self, args): + """ + # named_expr is also known as the "walrus op" := + expr ::= named_expr + named_expr ::= expr DUP_TOP store + """ + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): super(Python38Parser, self).__init__(debug_parser) self.customized = {} diff --git a/uncompyle6/semantics/customize38.py b/uncompyle6/semantics/customize38.py index 843ecf1c..4638c35d 100644 --- a/uncompyle6/semantics/customize38.py +++ b/uncompyle6/semantics/customize38.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2020 by Rocky Bernstein +# Copyright (c) 2019-2020, 2022 by Rocky Bernstein # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,133 +25,265 @@ def customize_for_version38(self, version): # FIXME: pytest doesn't add proper keys in testing. Reinstate after we have fixed pytest. # for lhs in 'for forelsestmt forelselaststmt ' - # 'forelselaststmtl tryfinally38'.split(): + # 'forelselaststmtc tryfinally38'.split(): # del TABLE_DIRECT[lhs] - TABLE_DIRECT.update({ - "async_for_stmt38": ( - "%|async for %c in %c:\n%+%c%-%-\n\n", - (2, "store"), (0, "expr"), (3, "for_block") ), - - 'async_forelse_stmt38': ( - '%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', - (7, 'store'), (0, 'expr'), (8, 'for_block'), (-1, 'else_suite') ), - - "async_with_stmt38": ( - "%|async with %c:\n%+%|%c%-", - (0, "expr"), 7), - - "async_with_as_stmt38": ( - "%|async with %c as %c:\n%+%|%c%-", - (0, "expr"), (6, "store"), - (7, "suite_stmts") - ), - - "except_cond_as": ( - "%|except %c as %c:\n", - (1, "expr"), - (-2, "STORE_FAST"), + TABLE_DIRECT.update( + { + "async_for_stmt38": ( + "%|async for %c in %c:\n%+%c%-%-\n\n", + (2, "store"), + (0, "expr"), + (3, "for_block"), ), + "async_forelse_stmt38": ( + "%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", + (7, 'store'), + (0, 'expr'), + (8, 'for_block'), + (-1, 'else_suite') + ), + "async_with_stmt38": ( + "%|async with %c:\n%+%c%-\n", + (0, "expr"), + (7, ("l_stmts_opt", "l_stmts", "pass")), + ), + "async_with_as_stmt38": ( + "%|async with %c as %c:\n%+%|%c%-", + (0, "expr"), + (6, "store"), + (7, "suite_stmts"), + ), + "c_forelsestmt38": ( + "%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", + (2, "store"), + (0, "expr"), + (3, "for_block"), + -1, + ), + "c_tryfinallystmt38": ( + "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", + (1, "c_suite_stmts_opt"), + (-2, "c_suite_stmts_opt"), + ), + "except_cond1a": ("%|except %c:\n", (1, "expr"),), + "except_cond_as": ( + "%|except %c as %c:\n", + (1, "expr"), + (-2, "STORE_FAST"), + ), + "except_handler38": ("%c", (2, "except_stmts")), + "except_handler38a": ("%c", (-2, "stmts")), + "except_handler38c": ( + "%c%+%c%-", + (1, "except_cond1a"), + (2, "except_stmts"), + ), + "except_handler_as": ( + "%c%+\n%+%c%-", + (1, "except_cond_as"), + (2, "tryfinallystmt"), + ), + "except_ret38a": ("return %c", (4, "expr")), + # Note: there is a suite_stmts_opt which seems + # to be bookkeeping which is not expressed in source code + "except_ret38": ("%|return %c\n", (1, "expr")), + "for38": ( + "%|for %c in %c:\n%+%c%-\n\n", + (2, "store"), + (0, "expr"), + (3, "for_block"), + ), + "forelsestmt38": ( + "%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", + (2, "store"), + (0, "expr"), + (3, "for_block"), + -1, + ), + "forelselaststmt38": ( + "%|for %c in %c:\n%+%c%-%|else:\n%+%c%-", + (2, "store"), + (0, "expr"), + (3, "for_block"), + -2, + ), + "forelselaststmtc38": ( + "%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", + (2, "store"), + (0, "expr"), + (3, "for_block"), + -2, + ), + "ifpoplaststmtc": ("%|if %c:\n%+%c%-", (0, "testexpr"), (2, "l_stmts")), + "pop_return": ("%|return %c\n", (1, "return_expr")), + "popb_return": ("%|return %c\n", (0, "return_expr")), + "pop_ex_return": ("%|return %c\n", (0, "return_expr")), + "set_for": (" for %c in %c", (2, "store"), (0, "expr_or_arg"),), + "whilestmt38": ( + "%|while %c:\n%+%c%-\n\n", + (1, "testexpr"), + (2, ("l_stmts", "pass")), + ), + "whileTruestmt38": ("%|while True:\n%+%c%-\n\n", (1, "l_stmts", "pass"),), + "try_elsestmtl38": ( + "%|try:\n%+%c%-%c%|else:\n%+%c%-", + (1, "suite_stmts_opt"), + (3, "except_handler38"), + (5, "else_suitel"), + ), + "try_except38": ( + "%|try:\n%+%c\n%-%|except:\n%+%c%-\n\n", + (2, ("suite_stmts_opt", "suite_stmts")), + (3, ("except_handler38a", "except_handler38b", "except_handler38c")), + ), + "try_except38r": ( + "%|try:\n%+%c\n%-%|except:\n%+%c%-\n\n", + (1, "return_except"), + (2, "except_handler38b"), + ), + "try_except38r2": ( + "%|try:\n%+%c\n%-%|except:\n%+%c%c%-\n\n", + (1, "suite_stmts_opt"), + (8, "cond_except_stmts_opt"), + (10, "return"), + ), + "try_except38r4": ( + "%|try:\n%+%c\n%-%|except:\n%+%c%c%-\n\n", + (1, "returns_in_except"), + (3, "except_cond1"), + (4, "return"), + ), + "try_except_as": ( + "%|try:\n%+%c%-\n%|%-%c\n\n", + ( + -4, + ("suite_stmts", "_stmts"), + ), # Go from the end because of POP_BLOCK variation + (-3, "except_handler_as"), + ), + "try_except_ret38": ( + "%|try:\n%+%c%-\n%|except:\n%+%|%c%-\n\n", + (1, "returns"), + (2, "except_ret38a"), + ), + "try_except_ret38a": ( + "%|try:\n%+%c%-%c\n\n", + (1, "returns"), + (2, "except_handler38c"), + ), + "tryfinally38rstmt": ( + "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", + (0, "sf_pb_call_returns"), + (-1, ("ss_end_finally", "suite_stmts", "_stmts")), + ), + "tryfinally38rstmt2": ( + "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", + (4, "returns"), + -2, + "ss_end_finally", + ), + "tryfinally38rstmt3": ( + "%|try:\n%+%|return %c%-\n%|finally:\n%+%c%-\n\n", + (1, "expr"), + (-1, "ss_end_finally"), + ), + "tryfinally38rstmt4": ( + "%|try:\n%+%c%-\n%|finally:\n%+%c%-\n\n", + (1, "suite_stmts_opt"), + (5, "suite_stmts_return"), + ), + "tryfinally38stmt": ( + "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", + (1, "suite_stmts_opt"), + (6, "suite_stmts_opt"), + ), + "tryfinally38astmt": ( + "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", + (2, "suite_stmts_opt"), + (8, "suite_stmts_opt"), + ), + "named_expr": ( # AKA "walrus operator" + "%c := %p", + (2, "store"), + (0, "expr", PRECEDENCE["named_expr"] - 1), + ), + } + ) - 'except_handler38': ( - '%c', (2, 'except_stmts') ), + def except_return_value(node): + if node[0] == "POP_BLOCK": + self.default(node[1]) + else: + self.template_engine(("%|return %c\n", (0, "expr")), node) + self.prune() - 'except_handler38a': ( - '%c', (-2, 'stmts') ), + self.n_except_return_value = except_return_value - "except_handler_as": ( - "%c%+\n%+%c%-", - (1, "except_cond_as"), - (2, "tryfinallystmt"), - ), + # FIXME: now that we've split out cond_except_stmt, + # we should be able to get this working as a pure transformation rule, + # so no procedure is needed here. + def try_except38r3(node): + self.template_engine(("%|try:\n%+%c\n%-", (1, "suite_stmts_opt")), node) + cond_except_stmts_opt = node[5] + assert cond_except_stmts_opt == "cond_except_stmts_opt" + for child in cond_except_stmts_opt: + if child == "cond_except_stmt": + if child[0] == "except_cond1": + self.template_engine( + ("%c\n", (0, "except_cond1"), (1, "expr")), child + ) + self.template_engine(("%+%c%-\n", (1, "except_stmts")), child) + pass + pass + self.template_engine(("%+%c%-\n", (7, "return")), node) + self.prune() - 'except_ret38a': ( - 'return %c', (4, 'expr') ), + self.n_try_except38r3 = try_except38r3 - # Note: there is a suite_stmts_opt which seems - # to be bookkeeping which is not expressed in source code - 'except_ret38': ( '%|return %c\n', (1, 'expr') ), - - 'for38': ( - '%|for %c in %c:\n%+%c%-\n\n', - (2, 'store'), - (0, 'expr'), - (3, 'for_block') ), - - "forelsestmt38": ( - "%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n", - (2, "store"), - (0, "expr"), - (3, "for_block"), -1 ), - - 'forelselaststmt38': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-', - (2, 'store'), - (0, 'expr'), - (3, 'for_block'), -2 ), - 'forelselaststmtl38': ( - '%|for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', - (2, 'store'), - (0, 'expr'), - (3, 'for_block'), -2 ), - - 'ifpoplaststmtl': ( '%|if %c:\n%+%c%-', - (0, "testexpr"), - (2, "c_stmts" ) ), - - 'ifstmtl': ( '%|if %c:\n%+%c%-', - (0, "testexpr"), - (1, "_ifstmts_jumpl") ), - - 'whilestmt38': ( '%|while %c:\n%+%c%-\n\n', - (1, 'testexpr'), - 2 ), # "l_stmts" or "pass" - 'whileTruestmt38': ( '%|while True:\n%+%c%-\n\n', - 1 ), # "l_stmts" or "pass" - 'try_elsestmtl38': ( - '%|try:\n%+%c%-%c%|else:\n%+%c%-', - (1, 'suite_stmts_opt'), - (3, 'except_handler38'), - (5, 'else_suitel') ), - 'try_except38': ( - '%|try:\n%+%c\n%-%|except:\n%|%-%c\n\n', - (-2, 'suite_stmts_opt'), (-1, 'except_handler38a') ), - - "try_except_as": ( - "%|try:\n%+%c%-\n%|%-%c\n\n", - (-4, "suite_stmts"), # Go from the end because of POP_BLOCK variation - (-3, "except_handler_as"), - ), - - "try_except_ret38": ( - "%|try:\n%+%c%-\n%|except:\n%+%|%c%-\n\n", - (1, "returns"), - (2, "except_ret38a"), - ), - 'tryfinally38rstmt': ( - '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', - (0, "sf_pb_call_returns"), - (-1, ("ss_end_finally", "suite_stmts")), - ), - "tryfinally38rstmt2": ( - "%|try:\n%+%c%-%|finally:\n%+%c%-\n\n", - (4, "returns"), - -2, "ss_end_finally" - ), - "tryfinally38rstmt3": ( - "%|try:\n%+%|return %c%-\n%|finally:\n%+%c%-\n\n", - (1, "expr"), - (-1, "ss_end_finally") - ), - 'tryfinally38stmt': ( - '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', - (1, "suite_stmts_opt"), - (6, "suite_stmts_opt") ), - 'tryfinally38astmt': ( - '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', - (2, "suite_stmts_opt"), - (8, "suite_stmts_opt") ), - "named_expr": ( # AKA "walrus operator" - "%c := %p", (2, "store"), (0, "expr", PRECEDENCE["named_expr"]-1) + def n_list_afor(node): + if len(node) == 2: + # list_afor ::= get_iter list_afor + self.comprehension_walk_newer(node, 0) + else: + list_iter_index = 2 if node[2] == "list_iter" else 3 + self.template_engine( + ( + " async for %[1]{%c} in %c%[1]{%c}", + (1, "store"), + (0, "get_aiter"), + (list_iter_index, "list_iter"), + ), + node, ) - }) + self.prune() + + self.n_list_afor = n_list_afor + + def n_set_afor(node): + if len(node) == 2: + self.template_engine( + (" async for %[1]{%c} in %c", (1, "store"), (0, "get_aiter")), node + ) + else: + self.template_engine( + " async for %[1]{%c} in %c%c", + (1, "store"), + (0, "get_aiter"), + (2, "set_iter"), + ) + self.prune() + + self.n_set_afor = n_set_afor + + def n_suite_stmts_return(node): + if len(node) > 1: + assert len(node) == 2 + self.template_engine( + ("%c\n%|return %c", (0, ("_stmts", "suite_stmts")), (1, "expr")), node + ) + else: + self.template_engine(("%|return %c", (0, "expr")), node) + self.prune() + + self.n_suite_stmts_return = n_suite_stmts_return