diff --git a/.travis.yml b/.travis.yml index d20842f2..c1357c5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,7 @@ language: python python: - - '2.7' # this is a cheat here because travis doesn't do 2.4-2.6 - -matrix: - include: - - python: '2.7' - dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069) + - 2.7 # this is a cheat here because travis doesn't do 2.4-2.6 install: - pip install -e . diff --git a/Makefile b/Makefile index c7f777ff..888e1df8 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ check-2.4 check-2.5: 2.6 5.0 5.3 5.6 5.8: #:PyPy pypy3-2.4.0 Python 3: -pypy-3.2 2.4: +7.1 pypy-3.2 2.4: $(MAKE) -C test $@ #: Run py.test tests diff --git a/NEWS.md b/NEWS.md index 259b34d6..3edf22f9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -4.1.0 2019-10-12 Stony Brook Ride +3.5.0 2019-10-12 Stony Brook Ride ================================= - Fix fragment bugs diff --git a/__pkginfo__.py b/__pkginfo__.py index 3a36b559..2e6a15de 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -58,7 +58,7 @@ entry_points = { ]} ftp_url = None install_requires = ["spark-parser >= 1.8.9, < 1.9.0", - "xdis >= 4.1.0, < 4.2.0"] + "xdis >= 4.1.2, < 4.2.0"] license = "GPL3" mailing_list = "python-debugger@googlegroups.com" diff --git a/admin-tools/how-to-make-a-release.md b/admin-tools/how-to-make-a-release.md index 976e6e81..81dcc4ad 100644 --- a/admin-tools/how-to-make-a-release.md +++ b/admin-tools/how-to-make-a-release.md @@ -63,6 +63,7 @@ Goto https://github.com/rocky/python-uncompyle6/releases # Upload single package and look at Rst Formating + $ twine check dist/uncompyle6-${VERSION}* $ twine upload dist/uncompyle6-${VERSION}-py3.3.egg # Upload rest of versions diff --git a/admin-tools/pyenv-newer-versions b/admin-tools/pyenv-newer-versions index 03900e8c..9c515556 100644 --- a/admin-tools/pyenv-newer-versions +++ b/admin-tools/pyenv-newer-versions @@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then echo "This script should be *sourced* rather than run directly through bash" exit 1 fi -export PYVERSIONS='3.6.9 3.7.4 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.10 3.5.7' +export PYVERSIONS='3.6.9 3.7.5 2.6.9 3.3.7 2.7.16 3.2.6 3.1.5 3.4.10 3.5.7' diff --git a/pytest/test_grammar.py b/pytest/test_grammar.py index 80f5e645..e2f3d2e7 100644 --- a/pytest/test_grammar.py +++ b/pytest/test_grammar.py @@ -1,50 +1,54 @@ import re -from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION +from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION from uncompyle6.parser import get_python_parser, python_parser from uncompyle6.scanner import get_scanner -def test_grammar(): +def test_grammar(): def check_tokens(tokens, opcode_set): remain_tokens = set(tokens) - opcode_set - remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens]) - remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens]) - remain_tokens = set([re.sub('LOAD_CODE$','', t) for t in remain_tokens]) + remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens]) + remain_tokens = set([re.sub("LOAD_CODE$", "", t) for t in remain_tokens]) remain_tokens = set(remain_tokens) - opcode_set - assert remain_tokens == set([]), \ - "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar()) + assert remain_tokens == set([]), "Remaining tokens %s\n====\n%s" % ( + remain_tokens, + p.dump_grammar(), + ) p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY) - (lhs, rhs, tokens, - right_recursive, dup_rhs) = p.check_sets() + (lhs, rhs, tokens, right_recursive, dup_rhs) = p.check_sets() # We have custom rules that create the below - expect_lhs = set(['pos_arg', 'attribute']) + expect_lhs = set(["pos_arg", "attribute"]) if PYTHON_VERSION < 3.8: - expect_lhs.add('get_iter') + expect_lhs.add("get_iter") + else: + expect_lhs.add("async_with_as_stmt") + expect_lhs.add("async_with_stmt") + unused_rhs = set(["list", "mkfunc", "mklambda", "unpack"]) - unused_rhs = set(['list', 'mkfunc', - 'mklambda', - 'unpack',]) - - expect_right_recursive = set([('designList', - ('store', 'DUP_TOP', 'designList'))]) + expect_right_recursive = set([("designList", ("store", "DUP_TOP", "designList"))]) if PYTHON_VERSION < 3.7: - unused_rhs.add('call') + unused_rhs.add("call") if PYTHON_VERSION > 2.6: - expect_lhs.add('kvlist') - expect_lhs.add('kv3') - unused_rhs.add('dict') + expect_lhs.add("kvlist") + expect_lhs.add("kv3") + unused_rhs.add("dict") if PYTHON3: - expect_lhs.add('load_genexpr') + expect_lhs.add("load_genexpr") - unused_rhs = unused_rhs.union(set(""" + unused_rhs = unused_rhs.union( + set( + """ except_pop_except generator_exp - """.split())) + """.split() + ) + ) if PYTHON_VERSION >= 3.0: expect_lhs.add("annotate_arg") expect_lhs.add("annotate_tuple") @@ -53,17 +57,19 @@ def test_grammar(): unused_rhs.add("classdefdeco1") unused_rhs.add("tryelsestmtl") if PYTHON_VERSION >= 3.5: - expect_right_recursive.add((('l_stmts', - ('lastl_stmt', 'come_froms', 'l_stmts')))) + expect_right_recursive.add( + (("l_stmts", ("lastl_stmt", "come_froms", "l_stmts"))) + ) pass elif 3.0 < PYTHON_VERSION < 3.3: - expect_right_recursive.add((('l_stmts', - ('lastl_stmt', 'COME_FROM', 'l_stmts')))) + expect_right_recursive.add( + (("l_stmts", ("lastl_stmt", "COME_FROM", "l_stmts"))) + ) pass pass pass else: - expect_lhs.add('kwarg') + expect_lhs.add("kwarg") assert expect_lhs == set(lhs) @@ -73,9 +79,16 @@ def test_grammar(): assert expect_right_recursive == right_recursive - expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',), - ('LOAD_CONST',), - ('JUMP_BACK',), ('JUMP_FORWARD',)]) + expect_dup_rhs = frozenset( + [ + ("COME_FROM",), + ("CONTINUE",), + ("JUMP_ABSOLUTE",), + ("LOAD_CONST",), + ("JUMP_BACK",), + ("JUMP_FORWARD",), + ] + ) reduced_dup_rhs = dict((k, dup_rhs[k]) for k in dup_rhs if k not in expect_dup_rhs) for k in reduced_dup_rhs: print(k, reduced_dup_rhs[k]) @@ -83,7 +96,7 @@ def test_grammar(): s = get_scanner(PYTHON_VERSION, IS_PYPY) ignore_set = set( - """ + """ JUMP_BACK CONTINUE COME_FROM COME_FROM_EXCEPT COME_FROM_EXCEPT_CLAUSE @@ -92,22 +105,33 @@ def test_grammar(): LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR LOAD_CODE LAMBDA_MARKER RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST - """.split()) + """.split() + ) if 2.6 <= PYTHON_VERSION <= 2.7: opcode_set = set(s.opc.opname).union(ignore_set) if PYTHON_VERSION == 2.6: opcode_set.add("THEN") check_tokens(tokens, opcode_set) elif PYTHON_VERSION == 3.4: - ignore_set.add('LOAD_CLASSNAME') - ignore_set.add('STORE_LOCALS') + ignore_set.add("LOAD_CLASSNAME") + ignore_set.add("STORE_LOCALS") opcode_set = set(s.opc.opname).union(ignore_set) check_tokens(tokens, opcode_set) + def test_dup_rule(): import inspect - python_parser(PYTHON_VERSION, inspect.currentframe().f_code, - is_pypy=IS_PYPY, - parser_debug={ - 'dups': True, 'transition': False, 'reduce': False, - 'rules': False, 'errorstack': None, 'context': True}) + + python_parser( + PYTHON_VERSION, + inspect.currentframe().f_code, + is_pypy=IS_PYPY, + parser_debug={ + "dups": True, + "transition": False, + "reduce": False, + "rules": False, + "errorstack": None, + "context": True, + }, + ) diff --git a/test/Makefile b/test/Makefile index 301451f8..b75e4c9d 100644 --- a/test/Makefile +++ b/test/Makefile @@ -100,7 +100,7 @@ check-bytecode-3: --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \ --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \ --bytecode-3.7 \ - --bytecode-pypy3.2 + --bytecode-pypy3.2 --bytecode-pypy3.6 --bytecode-3.8 #: Check deparsing on selected bytecode 3.x check-bytecode-3-short: @@ -338,7 +338,9 @@ pypy-3.2 2.4: $(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify #: PyPy 5.0.x with Python 3.6 ... +check-bytecode-pypy3.6: 7.1 7.1: + $(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run $(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify diff --git a/test/bytecode_3.2/11_classbug.pyc b/test/bytecode_3.2/11_classbug.pyc index 239bf2fc..342ee09b 100644 Binary files a/test/bytecode_3.2/11_classbug.pyc and b/test/bytecode_3.2/11_classbug.pyc differ diff --git a/test/bytecode_3.7/04_class_kwargs.pyc b/test/bytecode_3.7/04_class_kwargs.pyc index 1c095ac6..3cbcb72e 100644 Binary files a/test/bytecode_3.7/04_class_kwargs.pyc and b/test/bytecode_3.7/04_class_kwargs.pyc differ diff --git a/test/bytecode_3.8/01_for_continue.pyc b/test/bytecode_3.8/01_for_continue.pyc new file mode 100644 index 00000000..3f0ff90f Binary files /dev/null and b/test/bytecode_3.8/01_for_continue.pyc differ diff --git a/test/bytecode_pypy3.5/00_assign.pyc b/test/bytecode_pypy3.5/00_assign.pyc new file mode 100644 index 00000000..b6ad2b0a Binary files /dev/null and b/test/bytecode_pypy3.5/00_assign.pyc differ diff --git a/test/bytecode_pypy3.5/00_import.pyc b/test/bytecode_pypy3.5/00_import.pyc new file mode 100644 index 00000000..0198f100 Binary files /dev/null and b/test/bytecode_pypy3.5/00_import.pyc differ diff --git a/test/bytecode_pypy3.5/11_classbug.pyc b/test/bytecode_pypy3.5/11_classbug.pyc new file mode 100644 index 00000000..6c38f896 Binary files /dev/null and b/test/bytecode_pypy3.5/11_classbug.pyc differ diff --git a/test/bytecode_pypy3.6/00_import.pyc b/test/bytecode_pypy3.6/00_import.pyc new file mode 100644 index 00000000..e7f4ccfc Binary files /dev/null and b/test/bytecode_pypy3.6/00_import.pyc differ diff --git a/test/bytecode_pypy3.6/04_class_kwargs.pyc b/test/bytecode_pypy3.6/04_class_kwargs.pyc new file mode 100644 index 00000000..0a42c9e9 Binary files /dev/null and b/test/bytecode_pypy3.6/04_class_kwargs.pyc differ diff --git a/test/bytecode_pypy3.6/11_classbug.pyc b/test/bytecode_pypy3.6/11_classbug.pyc new file mode 100644 index 00000000..680e4364 Binary files /dev/null and b/test/bytecode_pypy3.6/11_classbug.pyc differ diff --git a/test/bytecode_pypy3.6_run/00_assign.pyc b/test/bytecode_pypy3.6_run/00_assign.pyc new file mode 100644 index 00000000..c711d3b4 Binary files /dev/null and b/test/bytecode_pypy3.6_run/00_assign.pyc differ diff --git a/test/bytecode_pypy3.6_run/00_docstring.pyc b/test/bytecode_pypy3.6_run/00_docstring.pyc new file mode 100644 index 00000000..d8cd26c8 Binary files /dev/null and b/test/bytecode_pypy3.6_run/00_docstring.pyc differ diff --git a/test/bytecode_pypy3.6_run/01_fstring.pyc b/test/bytecode_pypy3.6_run/01_fstring.pyc new file mode 100644 index 00000000..2151ae7b Binary files /dev/null and b/test/bytecode_pypy3.6_run/01_fstring.pyc differ diff --git a/test/simple_source/bug38/01_extra_iter.py b/test/simple_source/bug38/01_extra_iter.py new file mode 100644 index 00000000..476c2055 --- /dev/null +++ b/test/simple_source/bug38/01_extra_iter.py @@ -0,0 +1,7 @@ +# Adapted from From 3.3 urllib/parse.py +qs = "https://travis-ci.org/rocky/python-uncompyle6/builds/605260823?utm_medium=notification&utm_source=email" +expect = ['https://travis-ci.org/rocky/python-uncompyle6/builds/605260823?utm_medium=notification', 'utm_source=email'] + +# Should visually see that we don't add an extra iter() which is not technically wrong, just +# unnecessary. +assert expect == [s2 for s1 in qs.split('&') for s2 in s1.split(';')] diff --git a/test/simple_source/bug38/01_for_continue.py b/test/simple_source/bug38/01_for_continue.py new file mode 100644 index 00000000..7148a7ad --- /dev/null +++ b/test/simple_source/bug38/01_for_continue.py @@ -0,0 +1,5 @@ +# Bug is turning a JUMP_BACK for a CONTINUE so for has no JUMP_BACK. +# Also there is no POP_BLOCK since there isn't anything in the loop. +# In the future when we have better control flow, we might redo all of this. +for i in range(2): + pass diff --git a/test/test_pyenvlib.py b/test/test_pyenvlib.py index d4db5675..d01e3f71 100755 --- a/test/test_pyenvlib.py +++ b/test/test_pyenvlib.py @@ -42,6 +42,7 @@ TEST_VERSIONS = ( "pypy3.5-5.7.1-beta", "pypy3.5-5.9.0", "pypy3.5-6.0.0", + "pypy3.6-7.1.0", "native", ) + tuple(python_versions) diff --git a/uncompyle6/disas.py b/uncompyle6/disas.py index d603cef8..2ed115c9 100644 --- a/uncompyle6/disas.py +++ b/uncompyle6/disas.py @@ -32,12 +32,11 @@ want to run on earlier Python versions. import sys from collections import deque -import uncompyle6 - from xdis.code import iscode from xdis.load import check_object_path, load_module from uncompyle6.scanner import get_scanner + def disco(version, co, out=None, is_pypy=False): """ diassembles and deparses a given code block 'co' @@ -47,9 +46,9 @@ def disco(version, co, out=None, is_pypy=False): # store final output stream for case of error real_out = out or sys.stdout - real_out.write('# Python %s\n' % version) + real_out.write("# Python %s\n" % version) if co.co_filename: - real_out.write('# Embedded file name: %s\n' % co.co_filename) + real_out.write("# Embedded file name: %s\n" % co.co_filename) scanner = get_scanner(version, is_pypy=is_pypy) @@ -60,8 +59,8 @@ def disco(version, co, out=None, is_pypy=False): def disco_loop(disasm, queue, real_out): while len(queue) > 0: co = queue.popleft() - if co.co_name != '': - real_out.write('\n# %s line %d of %s\n' % + if co.co_name != "": + real_out.write("\n# %s line %d of %s\n" % (co.co_name, co.co_firstlineno, co.co_filename)) tokens, customize = disasm(co) for t in tokens: @@ -73,6 +72,7 @@ def disco_loop(disasm, queue, real_out): pass pass + # def disassemble_fp(fp, outstream=None): # """ # disassemble Python byte-code from an open file @@ -86,6 +86,7 @@ def disco_loop(disasm, queue, real_out): # disco(version, co, outstream, is_pypy=is_pypy) # co = None + def disassemble_file(filename, outstream=None): """ disassemble Python byte-code file (.pyc) @@ -94,8 +95,7 @@ def disassemble_file(filename, outstream=None): try to find the corresponding compiled object. """ filename = check_object_path(filename) - (version, timestamp, magic_int, co, is_pypy, - source_size) = load_module(filename) + (version, timestamp, magic_int, co, is_pypy, source_size) = load_module(filename) if type(co) == list: for con in co: disco(version, con, outstream) @@ -103,6 +103,7 @@ def disassemble_file(filename, outstream=None): disco(version, co, outstream, is_pypy=is_pypy) co = None + def _test(): """Simple test program to disassemble a file.""" argc = len(sys.argv) diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index fb1b5dd7..377ec715 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2018 Rocky Bernstein +# Copyright (c) 2015-2019 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # Copyright (c) 1999 John Aycock @@ -29,10 +29,10 @@ from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func from uncompyle6.parsers.treenode import SyntaxTree from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG -class Python2Parser(PythonParser): +class Python2Parser(PythonParser): def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): - super(Python2Parser, self).__init__(SyntaxTree, 'stmts', debug=debug_parser) + super(Python2Parser, self).__init__(SyntaxTree, "stmts", debug=debug_parser) self.new_rules = set() def p_print2(self, args): @@ -50,7 +50,7 @@ class Python2Parser(PythonParser): """ def p_print_to(self, args): - ''' + """ stmt ::= print_to stmt ::= print_to_nl stmt ::= print_nl_to @@ -60,10 +60,10 @@ class Python2Parser(PythonParser): print_to_items ::= print_to_items print_to_item print_to_items ::= print_to_item print_to_item ::= DUP_TOP expr ROT_TWO PRINT_ITEM_TO - ''' + """ def p_grammar(self, args): - ''' + """ sstmt ::= stmt sstmt ::= return RETURN_LAST @@ -174,12 +174,12 @@ class Python2Parser(PythonParser): jmp_abs ::= JUMP_ABSOLUTE jmp_abs ::= JUMP_BACK jmp_abs ::= CONTINUE - ''' + """ def p_generator_exp2(self, args): - ''' + """ generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 - ''' + """ def p_expr2(self, args): """ @@ -250,25 +250,41 @@ class Python2Parser(PythonParser): this. """ - if 'PyPy' in customize: + if "PyPy" in customize: # PyPy-specific customizations - self.addRule(""" + self.addRule( + """ stmt ::= assign3_pypy stmt ::= assign2_pypy assign3_pypy ::= expr expr expr store store store assign2_pypy ::= expr expr store store list_comp ::= expr BUILD_LIST_FROM_ARG for_iter store list_iter JUMP_BACK - """, nop_func) + """, + nop_func, + ) # For a rough break out on the first word. This may # include instructions that don't need customization, # but we'll do a finer check after the rough breakout. customize_instruction_basenames = frozenset( - ('BUILD', 'CALL', 'CONTINUE', 'DELETE', - 'DUP', 'EXEC', 'GET', 'JUMP', - 'LOAD', 'LOOKUP', 'MAKE', 'SETUP', - 'RAISE', 'UNPACK')) + ( + "BUILD", + "CALL", + "CONTINUE", + "DELETE", + "DUP", + "EXEC", + "GET", + "JUMP", + "LOAD", + "LOOKUP", + "MAKE", + "SETUP", + "RAISE", + "UNPACK", + ) + ) # Opcode names in the custom_seen_ops set have rules that get added # unconditionally and the rules are constant. So they need to be done @@ -282,139 +298,191 @@ class Python2Parser(PythonParser): # Do a quick breakout before testing potentially # each of the dozen or so instruction in if elif. - if (opname[:opname.find('_')] not in customize_instruction_basenames - or opname in custom_seen_ops): + if ( + opname[: opname.find("_")] not in customize_instruction_basenames + or opname in custom_seen_ops + ): continue - opname_base = opname[:opname.rfind('_')] + opname_base = opname[: opname.rfind("_")] # The order of opname listed is roughly sorted below - if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'): + if opname_base in ("BUILD_LIST", "BUILD_SET", "BUILD_TUPLE"): # We do this complicated test to speed up parsing of # pathelogically long literals, especially those over 1024. build_count = token.attr - thousands = (build_count//1024) - thirty32s = ((build_count//32) % 32) + thousands = build_count // 1024 + thirty32s = (build_count // 32) % 32 if thirty32s > 0: - rule = "expr32 ::=%s" % (' expr' * 32) + rule = "expr32 ::=%s" % (" expr" * 32) self.add_unique_rule(rule, opname_base, build_count, customize) if thousands > 0: - self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), - opname_base, build_count, customize) - collection = opname_base[opname_base.find('_')+1:].lower() - rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + - 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) - self.add_unique_rules([ - "expr ::= %s" % collection, - rule], customize) + self.add_unique_rule( + "expr1024 ::=%s" % (" expr32" * 32), + opname_base, + build_count, + customize, + ) + collection = opname_base[opname_base.find("_") + 1 :].lower() + rule = ( + ("%s ::= " % collection) + + "expr1024 " * thousands + + "expr32 " * thirty32s + + "expr " * (build_count % 32) + + opname + ) + self.add_unique_rules(["expr ::= %s" % collection, rule], customize) continue - elif opname_base == 'BUILD_MAP': - if opname == 'BUILD_MAP_n': + elif opname_base == "BUILD_MAP": + if opname == "BUILD_MAP_n": # PyPy sometimes has no count. Sigh. - self.add_unique_rules([ - 'kvlist_n ::= kvlist_n kv3', - 'kvlist_n ::=', - 'dict ::= BUILD_MAP_n kvlist_n', - ], customize) + self.add_unique_rules( + [ + "kvlist_n ::= kvlist_n kv3", + "kvlist_n ::=", + "dict ::= BUILD_MAP_n kvlist_n", + ], + customize, + ) if self.version >= 2.7: self.add_unique_rule( - 'dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store ' - 'comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST', - 'dict_comp_func', 0, customize) + "dict_comp_func ::= BUILD_MAP_n LOAD_FAST FOR_ITER store " + "comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST", + "dict_comp_func", + 0, + customize, + ) else: - kvlist_n = ' kv3' * token.attr + kvlist_n = " kv3" * token.attr rule = "dict ::= %s%s" % (opname, kvlist_n) self.addRule(rule, nop_func) continue - elif opname_base == 'BUILD_SLICE': - slice_num = token.attr + elif opname_base == "BUILD_SLICE": + slice_num = token.attr if slice_num == 2: - self.add_unique_rules([ - 'expr ::= build_slice2', - 'build_slice2 ::= expr expr BUILD_SLICE_2' - ], customize) + self.add_unique_rules( + [ + "expr ::= build_slice2", + "build_slice2 ::= expr expr BUILD_SLICE_2", + ], + customize, + ) else: - assert slice_num == 3, ("BUILD_SLICE value must be 2 or 3; is %s" % - slice_num) - self.add_unique_rules([ - 'expr ::= build_slice3', - 'build_slice3 ::= expr expr expr BUILD_SLICE_3', - ], customize) + assert slice_num == 3, ( + "BUILD_SLICE value must be 2 or 3; is %s" % slice_num + ) + self.add_unique_rules( + [ + "expr ::= build_slice3", + "build_slice3 ::= expr expr expr BUILD_SLICE_3", + ], + customize, + ) continue - elif opname_base in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', - 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): + elif opname_base in ( + "CALL_FUNCTION", + "CALL_FUNCTION_VAR", + "CALL_FUNCTION_VAR_KW", + "CALL_FUNCTION_KW", + ): args_pos, args_kw = self.get_pos_kw(token) # number of apply equiv arguments: - nak = ( len(opname_base)-len('CALL_FUNCTION') ) // 3 - rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ - + 'expr ' * nak + opname - elif opname_base == 'CALL_METHOD': + nak = (len(opname_base) - len("CALL_FUNCTION")) // 3 + rule = ( + "call ::= expr " + + "expr " * args_pos + + "kwarg " * args_kw + + "expr " * nak + + opname + ) + elif opname_base == "CALL_METHOD": # PyPy only - DRY with parse3 args_pos, args_kw = self.get_pos_kw(token) # number of apply equiv arguments: - nak = ( len(opname_base)-len('CALL_METHOD') ) // 3 - rule = 'call ::= expr ' + 'expr '*args_pos + 'kwarg '*args_kw \ - + 'expr ' * nak + opname - elif opname == 'CONTINUE_LOOP': - self.addRule('continue ::= CONTINUE_LOOP', nop_func) + nak = (len(opname_base) - len("CALL_METHOD")) // 3 + rule = ( + "call ::= expr " + + "expr " * args_pos + + "kwarg " * args_kw + + "expr " * nak + + opname + ) + elif opname == "CONTINUE_LOOP": + self.addRule("continue ::= CONTINUE_LOOP", nop_func) custom_seen_ops.add(opname) continue - elif opname == 'DELETE_ATTR': - self.addRule('del_stmt ::= expr DELETE_ATTR', nop_func) + elif opname == "DELETE_ATTR": + self.addRule("del_stmt ::= expr DELETE_ATTR", nop_func) custom_seen_ops.add(opname) continue - elif opname.startswith('DELETE_SLICE'): - self.addRule(""" + elif opname.startswith("DELETE_SLICE"): + self.addRule( + """ del_expr ::= expr del_stmt ::= del_expr DELETE_SLICE+0 del_stmt ::= del_expr del_expr DELETE_SLICE+1 del_stmt ::= del_expr del_expr DELETE_SLICE+2 del_stmt ::= del_expr del_expr del_expr DELETE_SLICE+3 - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) - self.check_reduce['del_expr'] = 'AST' + self.check_reduce["del_expr"] = "AST" continue - elif opname == 'DELETE_DEREF': - self.addRule(""" + elif opname == "DELETE_DEREF": + self.addRule( + """ stmt ::= del_deref_stmt del_deref_stmt ::= DELETE_DEREF - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname == 'DELETE_SUBSCR': - self.addRule(""" + elif opname == "DELETE_SUBSCR": + self.addRule( + """ del_stmt ::= delete_subscript delete_subscript ::= expr expr DELETE_SUBSCR - """, nop_func) - self.check_reduce['delete_subscript'] = 'AST' + """, + nop_func, + ) + self.check_reduce["delete_subscript"] = "AST" custom_seen_ops.add(opname) continue - elif opname == 'GET_ITER': - self.addRule(""" + elif opname == "GET_ITER": + self.addRule( + """ expr ::= get_iter attribute ::= expr GET_ITER - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname_base in ('DUP_TOPX', 'RAISE_VARARGS'): + elif opname_base in ("DUP_TOPX", "RAISE_VARARGS"): # FIXME: remove these conditions if they are not needed. # no longer need to add a rule continue - elif opname == 'EXEC_STMT': - self.addRule(""" + elif opname == "EXEC_STMT": + self.addRule( + """ stmt ::= exec_stmt exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT exec_stmt ::= expr exprlist EXEC_STMT exprlist ::= expr+ - """, nop_func) + """, + nop_func, + ) continue - elif opname == 'JUMP_IF_NOT_DEBUG': - self.addRule(""" + elif opname == "JUMP_IF_NOT_DEBUG": + self.addRule( + """ jmp_true_false ::= POP_JUMP_IF_TRUE jmp_true_false ::= POP_JUMP_IF_FALSE stmt ::= assert_pypy @@ -424,107 +492,152 @@ class Python2Parser(PythonParser): assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true_false LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM - """, nop_func) + """, + nop_func, + ) continue - elif opname == 'LOAD_ATTR': - self.addRule(""" + elif opname == "LOAD_ATTR": + self.addRule( + """ expr ::= attribute attribute ::= expr LOAD_ATTR - """, nop_func) + """, + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname == 'LOAD_LISTCOMP': + elif opname == "LOAD_LISTCOMP": self.addRule("expr ::= listcomp", nop_func) custom_seen_ops.add(opname) continue - elif opname == 'LOAD_SETCOMP': - self.add_unique_rules([ - "expr ::= set_comp", - "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1" - ], customize) + elif opname == "LOAD_SETCOMP": + self.add_unique_rules( + [ + "expr ::= set_comp", + "set_comp ::= LOAD_SETCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1", + ], + customize, + ) custom_seen_ops.add(opname) continue - elif opname == 'LOOKUP_METHOD': + elif opname == "LOOKUP_METHOD": # A PyPy speciality - DRY with parse3 - self.addRule(""" + self.addRule( + """ expr ::= attribute attribute ::= expr LOOKUP_METHOD """, - nop_func) + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname_base == 'MAKE_FUNCTION': - if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': - self.addRule('mklambda ::= %s LOAD_LAMBDA %s' % - ('pos_arg ' * token.attr, opname), nop_func) - rule = 'mkfunc ::= %s LOAD_CODE %s' % ('expr ' * token.attr, opname) - elif opname_base == 'MAKE_CLOSURE': + elif opname_base == "MAKE_FUNCTION": + if i > 0 and tokens[i - 1] == "LOAD_LAMBDA": + self.addRule( + "mklambda ::= %s LOAD_LAMBDA %s" + % ("pos_arg " * token.attr, opname), + nop_func, + ) + rule = "mkfunc ::= %s LOAD_CODE %s" % ("expr " * token.attr, opname) + elif opname_base == "MAKE_CLOSURE": # FIXME: use add_unique_rules to tidy this up. - if i > 0 and tokens[i-1] == 'LOAD_LAMBDA': - self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' % - ('expr ' * token.attr, opname), nop_func) + if i > 0 and tokens[i - 1] == "LOAD_LAMBDA": + self.addRule( + "mklambda ::= %s load_closure LOAD_LAMBDA %s" + % ("expr " * token.attr, opname), + nop_func, + ) if i > 0: - prev_tok = tokens[i-1] - if prev_tok == 'LOAD_GENEXPR': - self.add_unique_rules([ - ('generator_exp ::= %s load_closure LOAD_GENEXPR %s expr' - ' GET_ITER CALL_FUNCTION_1' % - ('expr ' * token.attr, opname))], customize) + prev_tok = tokens[i - 1] + if prev_tok == "LOAD_GENEXPR": + self.add_unique_rules( + [ + ( + "generator_exp ::= %s load_closure LOAD_GENEXPR %s expr" + " GET_ITER CALL_FUNCTION_1" + % ("expr " * token.attr, opname) + ) + ], + customize, + ) pass - self.add_unique_rules([ - ('mkfunc ::= %s load_closure LOAD_CODE %s' % - ('expr ' * token.attr, opname))], customize) + self.add_unique_rules( + [ + ( + "mkfunc ::= %s load_closure LOAD_CODE %s" + % ("expr " * token.attr, opname) + ) + ], + customize, + ) if self.version >= 2.7: if i > 0: - prev_tok = tokens[i-1] - if prev_tok == 'LOAD_DICTCOMP': - self.add_unique_rules([ - ('dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr' - ' GET_ITER CALL_FUNCTION_1' % - ('expr ' * token.attr, opname))], customize) - elif prev_tok == 'LOAD_SETCOMP': - self.add_unique_rules([ - "expr ::= set_comp", - ('set_comp ::= %s load_closure LOAD_SETCOMP %s expr' - ' GET_ITER CALL_FUNCTION_1' % - ('expr ' * token.attr, opname)) - ], customize) + prev_tok = tokens[i - 1] + if prev_tok == "LOAD_DICTCOMP": + self.add_unique_rules( + [ + ( + "dict_comp ::= %s load_closure LOAD_DICTCOMP %s expr" + " GET_ITER CALL_FUNCTION_1" + % ("expr " * token.attr, opname) + ) + ], + customize, + ) + elif prev_tok == "LOAD_SETCOMP": + self.add_unique_rules( + [ + "expr ::= set_comp", + ( + "set_comp ::= %s load_closure LOAD_SETCOMP %s expr" + " GET_ITER CALL_FUNCTION_1" + % ("expr " * token.attr, opname) + ), + ], + customize, + ) pass pass continue - elif opname == 'SETUP_EXCEPT': - if 'PyPy' in customize: - self.add_unique_rules([ - "stmt ::= try_except_pypy", - "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy", - "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM" - ], customize) + elif opname == "SETUP_EXCEPT": + if "PyPy" in customize: + self.add_unique_rules( + [ + "stmt ::= try_except_pypy", + "try_except_pypy ::= SETUP_EXCEPT suite_stmts_opt except_handler_pypy", + "except_handler_pypy ::= COME_FROM except_stmts END_FINALLY COME_FROM", + ], + customize, + ) custom_seen_ops.add(opname) continue - elif opname == 'SETUP_FINALLY': - if 'PyPy' in customize: - self.addRule(""" + elif opname == "SETUP_FINALLY": + if "PyPy" in customize: + self.addRule( + """ stmt ::= tryfinallystmt_pypy tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY - suite_stmts_opt END_FINALLY""", nop_func) + suite_stmts_opt END_FINALLY""", + nop_func, + ) custom_seen_ops.add(opname) continue - elif opname_base in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'): + elif opname_base in ("UNPACK_TUPLE", "UNPACK_SEQUENCE"): custom_seen_ops.add(opname) - rule = 'unpack ::= ' + opname + ' store' * token.attr - elif opname_base == 'UNPACK_LIST': + rule = "unpack ::= " + opname + " store" * token.attr + elif opname_base == "UNPACK_LIST": custom_seen_ops.add(opname) - rule = 'unpack_list ::= ' + opname + ' store' * token.attr + rule = "unpack_list ::= " + opname + " store" * token.attr else: continue self.addRule(rule, nop_func) pass - self.check_reduce['raise_stmt1'] = 'tokens' - self.check_reduce['aug_assign2'] = 'AST' - self.check_reduce['or'] = 'AST' + self.check_reduce["raise_stmt1"] = "tokens" + self.check_reduce["aug_assign2"] = "AST" + self.check_reduce["or"] = "AST" # self.check_reduce['_stmts'] = 'AST' # Dead code testing... @@ -539,24 +652,30 @@ class Python2Parser(PythonParser): # Dead code testing... # if lhs == 'while1elsestmt': # from trepan.api import debug; debug() - if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'): + if ( + lhs in ("aug_assign1", "aug_assign2") + and ast[0] + and ast[0][0] in ("and", "or") + ): return True - elif lhs in ('raise_stmt1',): + elif lhs in ("raise_stmt1",): # We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule - return (tokens[first] == 'LOAD_ASSERT' and (last >= len(tokens))) - elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')): + return tokens[first] == "LOAD_ASSERT" and (last >= len(tokens)) + elif rule == ("or", ("expr", "jmp_true", "expr", "\\e_come_from_opt")): expr2 = ast[2] - return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT' - elif lhs in ('delete_subscript', 'del_expr'): + return expr2 == "expr" and expr2[0] == "LOAD_ASSERT" + elif lhs in ("delete_subscript", "del_expr"): op = ast[0][0] - return op.kind in ('and', 'or') + return op.kind in ("and", "or") return False + class Python2ParserSingle(Python2Parser, PythonParserSingle): pass -if __name__ == '__main__': + +if __name__ == "__main__": # Check grammar p = Python2Parser() p.check_grammar() diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 378b7cda..b078f7e4 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -432,7 +432,7 @@ class Python3Parser(PythonParser): else: return "%s_0" % (token.kind) - def custom_build_class_rule(self, opname, i, token, tokens, customize): + def custom_build_class_rule(self, opname, i, token, tokens, customize, is_pypy): """ # Should the first rule be somehow folded into the 2nd one? build_class ::= LOAD_BUILD_CLASS mkfunc @@ -485,10 +485,18 @@ class Python3Parser(PythonParser): call_function = call_fn_tok.kind if call_function.startswith("CALL_FUNCTION_KW"): self.addRule("classdef ::= build_class_kw store", nop_func) - rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" % ( - "expr " * (call_fn_tok.attr - 1), - call_function, - ) + if is_pypy: + args_pos, args_kw = self.get_pos_kw(call_fn_tok) + rule = "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %s%s%s" % ( + "expr " * (args_pos - 1), + "kwarg " * (args_kw), + call_function, + ) + else: + rule = ( + "build_class_kw ::= LOAD_BUILD_CLASS mkfunc %sLOAD_CONST %s" + % ("expr " * (call_fn_tok.attr - 1), call_function) + ) else: call_function = self.call_fn_name(call_fn_tok) rule = "build_class ::= LOAD_BUILD_CLASS mkfunc %s%s" % ( @@ -498,7 +506,7 @@ class Python3Parser(PythonParser): self.addRule(rule, nop_func) return - def custom_classfunc_rule(self, opname, token, customize, next_token): + def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy): """ call ::= expr {expr}^n CALL_FUNCTION_n call ::= expr {expr}^n CALL_FUNCTION_VAR_n @@ -516,18 +524,28 @@ class Python3Parser(PythonParser): # Yes, this computation based on instruction name is a little bit hoaky. nak = (len(opname) - len("CALL_FUNCTION")) // 3 - token.kind = self.call_fn_name(token) uniq_param = args_kw + args_pos # Note: 3.5+ have subclassed this method; so we don't handle # 'CALL_FUNCTION_VAR' or 'CALL_FUNCTION_EX' here. - rule = ( - "call ::= expr " - + ("pos_arg " * args_pos) - + ("kwarg " * args_kw) - + "expr " * nak - + token.kind - ) + if is_pypy and self.version >= 3.6: + if token == "CALL_FUNCTION": + token.kind = self.call_fn_name(token) + rule = ( + "call ::= expr " + + ("pos_arg " * args_pos) + + ("kwarg " * args_kw) + + token.kind + ) + else: + token.kind = self.call_fn_name(token) + rule = ( + "call ::= expr " + + ("pos_arg " * args_pos) + + ("kwarg " * args_kw) + + "expr " * nak + + token.kind + ) self.add_unique_rule(rule, token.kind, uniq_param, customize) @@ -545,7 +563,12 @@ class Python3Parser(PythonParser): this has an effect on many rules. """ if self.version >= 3.3: - new_rule = rule % (("LOAD_STR ") * 1) + if PYTHON3 or not self.is_pypy: + load_op = "LOAD_STR " + else: + load_op = "LOAD_CONST " + + new_rule = rule % ((load_op) * 1) else: new_rule = rule % (("LOAD_STR ") * 0) self.add_unique_rule(new_rule, opname, attr, customize) @@ -573,7 +596,7 @@ class Python3Parser(PythonParser): """ - is_pypy = False + self.is_pypy = False # For a rough break out on the first word. This may # include instructions that don't need customization, @@ -618,7 +641,7 @@ class Python3Parser(PythonParser): # a specific instruction seen. if "PyPy" in customize: - is_pypy = True + self.is_pypy = True self.addRule( """ stmt ::= assign3_pypy @@ -823,7 +846,9 @@ class Python3Parser(PythonParser): """ self.addRule(rule, nop_func) - self.custom_classfunc_rule(opname, token, customize, tokens[i + 1]) + self.custom_classfunc_rule( + opname, token, customize, tokens[i + 1], self.is_pypy + ) # Note: don't add to custom_ops_processed. elif opname_base == "CALL_METHOD": @@ -882,21 +907,30 @@ class Python3Parser(PythonParser): self.addRule( """ stmt ::= assert_pypy - stmt ::= assert2_pypy", nop_func) + stmt ::= assert_not_pypy + stmt ::= assert2_pypy + stmt ::= assert2_not_pypy assert_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM + assert_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false + LOAD_ASSERT RAISE_VARARGS_1 COME_FROM assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1 COME_FROM assert2_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 - RAISE_VARARGS_1 COME_FROM, + RAISE_VARARGS_1 COME_FROM + assert2_not_pypy ::= JUMP_IF_NOT_DEBUG assert_expr jmp_false + LOAD_ASSERT expr CALL_FUNCTION_1 + RAISE_VARARGS_1 COME_FROM """, nop_func, ) custom_ops_processed.add(opname) elif opname == "LOAD_BUILD_CLASS": - self.custom_build_class_rule(opname, i, token, tokens, customize) + self.custom_build_class_rule( + opname, i, token, tokens, customize, self.is_pypy + ) # Note: don't add to custom_ops_processed. elif opname == "LOAD_CLASSDEREF": # Python 3.4+ @@ -969,7 +1003,7 @@ class Python3Parser(PythonParser): j = 1 else: j = 2 - if is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"): + if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"): rule_pat = "mklambda ::= %sload_closure LOAD_LAMBDA %%s%s" % ( "pos_arg " * args_pos, opname, @@ -984,7 +1018,7 @@ class Python3Parser(PythonParser): self.add_make_function_rule(rule_pat, opname, token.attr, customize) if has_get_iter_call_function1: - if is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"): + if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"): # In the tokens we saw: # LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or # LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or @@ -998,7 +1032,7 @@ class Python3Parser(PythonParser): self.add_make_function_rule( rule_pat, opname, token.attr, customize ) - if is_pypy or (i >= j and tokens[i - j] == "LOAD_SETCOMP"): + if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_SETCOMP"): rule_pat = ( "set_comp ::= %sload_closure LOAD_SETCOMP %%s%s expr " "GET_ITER CALL_FUNCTION_1" @@ -1007,7 +1041,7 @@ class Python3Parser(PythonParser): self.add_make_function_rule( rule_pat, opname, token.attr, customize ) - if is_pypy or (i >= j and tokens[i - j] == "LOAD_DICTCOMP"): + if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_DICTCOMP"): self.add_unique_rule( "dict_comp ::= %sload_closure LOAD_DICTCOMP %s " "expr GET_ITER CALL_FUNCTION_1" @@ -1053,17 +1087,24 @@ class Python3Parser(PythonParser): ) elif self.version >= 3.4: + if PYTHON3 or not self.is_pypy: + load_op = "LOAD_STR" + else: + load_op = "LOAD_CONST" + if annotate_args > 0: - rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s" % ( + rule = "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure %s %s %s" % ( "pos_arg " * args_pos, kwargs_str, "annotate_arg " * (annotate_args - 1), + load_op, opname, ) else: - rule = "mkfunc ::= %s%s load_closure LOAD_CODE LOAD_STR %s" % ( + rule = "mkfunc ::= %s%s load_closure LOAD_CODE %s %s" % ( "pos_arg " * args_pos, kwargs_str, + load_op, opname, ) @@ -1121,6 +1162,14 @@ class Python3Parser(PythonParser): opname, ) self.add_unique_rule(rule, opname, token.attr, customize) + if not PYTHON3 and self.is_pypy: + rule = "mkfunc ::= %s%s%s%s" % ( + "expr " * stack_count, + "load_closure " * closure, + "LOAD_CODE LOAD_CONST ", + opname, + ) + self.add_unique_rule(rule, opname, token.attr, customize) if has_get_iter_call_function1: rule_pat = ( @@ -1137,7 +1186,7 @@ class Python3Parser(PythonParser): self.add_make_function_rule( rule_pat, opname, token.attr, customize ) - if is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP"): + if self.is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LISTCOMP"): if self.version >= 3.6: # 3.6+ sometimes bundles all of the # 'exprs' in the rule above into a @@ -1158,7 +1207,7 @@ class Python3Parser(PythonParser): rule_pat, opname, token.attr, customize ) - if is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LAMBDA"): + if self.is_pypy or (i >= 2 and tokens[i - 2] == "LOAD_LAMBDA"): rule_pat = "mklambda ::= %s%sLOAD_LAMBDA %%s%s" % ( ("pos_arg " * args_pos), ("kwarg " * args_kw), @@ -1186,7 +1235,7 @@ class Python3Parser(PythonParser): ) self.add_make_function_rule(rule_pat, opname, token.attr, customize) - if is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"): + if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LISTCOMP"): # In the tokens we saw: # LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION (>= 3.3) or # LOAD_LISTCOMP MAKE_FUNCTION (< 3.3) or @@ -1201,7 +1250,7 @@ class Python3Parser(PythonParser): ) # FIXME: Fold test into add_make_function_rule - if is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"): + if self.is_pypy or (i >= j and tokens[i - j] == "LOAD_LAMBDA"): rule_pat = "mklambda ::= %s%sLOAD_LAMBDA %%s%s" % ( ("pos_arg " * args_pos), ("kwarg " * args_kw), diff --git a/uncompyle6/parsers/parse35.py b/uncompyle6/parsers/parse35.py index e017484b..32488702 100644 --- a/uncompyle6/parsers/parse35.py +++ b/uncompyle6/parsers/parse35.py @@ -143,9 +143,15 @@ class Python35Parser(Python34Parser): super(Python35Parser, self).customize_grammar_rules(tokens, customize) for i, token in enumerate(tokens): opname = token.kind + if opname == 'LOAD_ASSERT': + if 'PyPy' in customize: + rules_str = """ + stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM + """ + self.add_unique_doc_rules(rules_str, customize) # FIXME: I suspect this is wrong for 3.6 and 3.5, but # I haven't verified what the 3.7ish fix is - if opname == 'BUILD_MAP_UNPACK_WITH_CALL': + elif opname == 'BUILD_MAP_UNPACK_WITH_CALL': if self.version < 3.7: self.addRule("expr ::= unmapexpr", nop_func) nargs = token.attr % 256 diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py index 3cff38ce..42fc9ec2 100644 --- a/uncompyle6/parsers/parse36.py +++ b/uncompyle6/parsers/parse36.py @@ -187,13 +187,7 @@ class Python36Parser(Python35Parser): for i, token in enumerate(tokens): opname = token.kind - if opname == 'LOAD_ASSERT': - if 'PyPy' in customize: - rules_str = """ - stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM - """ - self.add_unique_doc_rules(rules_str, customize) - elif opname == 'FORMAT_VALUE': + if opname == 'FORMAT_VALUE': rules_str = """ expr ::= formatted_value1 formatted_value1 ::= expr FORMAT_VALUE @@ -315,7 +309,7 @@ class Python36Parser(Python35Parser): pass return - def custom_classfunc_rule(self, opname, token, customize, next_token): + def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy): args_pos, args_kw = self.get_pos_kw(token) @@ -337,10 +331,14 @@ class Python36Parser(Python35Parser): self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize) if opname.startswith('CALL_FUNCTION_KW'): - self.addRule("expr ::= call_kw36", nop_func) - values = 'expr ' * token.attr - rule = "call_kw36 ::= expr %s LOAD_CONST %s" % (values, opname) - self.add_unique_rule(rule, token.kind, token.attr, customize) + if is_pypy: + # PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions + super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy) + else: + self.addRule("expr ::= call_kw36", nop_func) + values = 'expr ' * token.attr + rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals()) + self.add_unique_rule(rule, token.kind, token.attr, customize) elif opname == 'CALL_FUNCTION_EX_KW': # Note: this doesn't exist in 3.7 and later self.addRule("""expr ::= call_ex_kw4 @@ -405,7 +403,7 @@ class Python36Parser(Python35Parser): """, nop_func) pass else: - super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token) + super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy) def reduce_is_invalid(self, rule, ast, tokens, first, last): invalid = super(Python36Parser, diff --git a/uncompyle6/parsers/parse38.py b/uncompyle6/parsers/parse38.py index f6a2c0d5..b137385e 100644 --- a/uncompyle6/parsers/parse38.py +++ b/uncompyle6/parsers/parse38.py @@ -92,6 +92,7 @@ class Python38Parser(Python37Parser): for38 ::= expr get_iter store for_block JUMP_BACK for38 ::= expr for_iter store for_block JUMP_BACK for38 ::= expr for_iter store for_block JUMP_BACK POP_BLOCK + for38 ::= expr for_iter store for_block forelsestmt38 ::= expr for_iter store for_block POP_BLOCK else_suite forelselaststmt38 ::= expr for_iter store for_block POP_BLOCK else_suitec diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 641d1293..90543da8 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -27,8 +27,7 @@ import sys from uncompyle6 import PYTHON3, IS_PYPY, PYTHON_VERSION from uncompyle6.scanners.tok import Token import xdis -from xdis.bytecode import ( - Bytecode, instruction_size, extended_arg_val, next_offset) +from xdis.bytecode import Bytecode, instruction_size, extended_arg_val, next_offset from xdis.magics import canonic_python_version from xdis.util import code2num @@ -39,15 +38,38 @@ else: # The byte code versions we support. # Note: these all have to be floats -PYTHON_VERSIONS = frozenset((1.0, 1.1, 1.3, 1.4, 1.5, 1.6, - 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, - 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8)) +PYTHON_VERSIONS = frozenset( + ( + 1.0, + 1.1, + 1.3, + 1.4, + 1.5, + 1.6, + 2.1, + 2.2, + 2.3, + 2.4, + 2.5, + 2.6, + 2.7, + 3.0, + 3.1, + 3.2, + 3.3, + 3.4, + 3.5, + 3.6, + 3.7, + 3.8, + ) +) CANONIC2VERSION = dict((canonic_python_version[str(v)], v) for v in PYTHON_VERSIONS) # Magic changed mid version for Python 3.5.2. Compatibility was added for # the older 3.5 interpreter magic. -CANONIC2VERSION['3.5.2'] = 3.5 +CANONIC2VERSION["3.5.2"] = 3.5 # FIXME: DRY @@ -57,24 +79,28 @@ if PYTHON3: def long(l): return l + + else: - L65536 = long(65536) # NOQA + L65536 = long(65536) # NOQA + class Code(object): - ''' + """ Class for representing code-objects. This is similar to the original code object, but additionally the diassembled code is stored in the attribute '_tokens'. - ''' + """ + def __init__(self, co, scanner, classname=None): for i in dir(co): - if i.startswith('co_'): + if i.startswith("co_"): setattr(self, i, getattr(co, i)) self._tokens, self._customize = scanner.ingest(co, classname) -class Scanner(object): +class Scanner(object): def __init__(self, version, show_asm=None, is_pypy=False): self.version = version self.show_asm = show_asm @@ -102,7 +128,7 @@ class Scanner(object): """ # FIXME: remove this when all subsidiary functions have been removed. # We should be able to get everything from the self.insts list. - self.code = array('B', co.co_code) + self.code = array("B", co.co_code) bytecode = Bytecode(co, self.opc) self.build_prev_op() @@ -130,7 +156,7 @@ class Scanner(object): # 'List-map' which shows line number of current op and offset of # first op on following line, given offset of op as index lines = [] - LineTuple = namedtuple('LineTuple', ['l_no', 'next']) + LineTuple = namedtuple("LineTuple", ["l_no", "next"]) # Iterate through available linestarts, and fill # the data for all code offsets encountered until @@ -173,14 +199,14 @@ class Scanner(object): goes forward. """ opname = self.get_inst(offset).opname - if opname == 'JUMP_FORWARD': + if opname == "JUMP_FORWARD": return True - if opname != 'JUMP_ABSOLUTE': + if opname != "JUMP_ABSOLUTE": return False return offset < self.get_target(offset) def prev_offset(self, offset): - return self.insts[self.offset2inst_index[offset]-1].offset + return self.insts[self.offset2inst_index[offset] - 1].offset def get_inst(self, offset): # Instructions can get moved as a result of EXTENDED_ARGS removal. @@ -207,7 +233,7 @@ class Scanner(object): return target def get_argument(self, pos): - arg = self.code[pos+1] + self.code[pos+2] * 256 + arg = self.code[pos + 1] + self.code[pos + 2] * 256 return arg def next_offset(self, op, offset): @@ -218,9 +244,9 @@ class Scanner(object): op = self.code[i] if op in self.JUMP_OPS: dest = self.get_target(i, op) - print('%i\t%s\t%i' % (i, self.opname[op], dest)) + print("%i\t%s\t%i" % (i, self.opname[op], dest)) else: - print('%i\t%s\t' % (i, self.opname[op])) + print("%i\t%s\t" % (i, self.opname[op])) def first_instr(self, start, end, instr, target=None, exact=True): """ @@ -234,11 +260,9 @@ class Scanner(object): Return index to it or None if not found. """ code = self.code - assert(start >= 0 and end <= len(code)) + assert start >= 0 and end <= len(code) - try: - None in instr - except: + if not isinstance(instr, list): instr = [instr] result_offset = None @@ -276,9 +300,7 @@ class Scanner(object): if not (start >= 0 and end <= len(code)): return None - try: - None in instr - except: + if not isinstance(instr, list): instr = [instr] result_offset = None @@ -289,7 +311,7 @@ class Scanner(object): op = code[offset] if op == self.opc.EXTENDED_ARG: - arg = code2num(code, offset+1) | extended_arg + arg = code2num(code, offset + 1) | extended_arg extended_arg = extended_arg_val(self.opc, arg) continue @@ -367,7 +389,7 @@ class Scanner(object): """ code = self.code - assert(start >= 0 and end <= len(code)) + assert start >= 0 and end <= len(code) try: None in instr @@ -381,7 +403,7 @@ class Scanner(object): op = code[offset] if op == self.opc.EXTENDED_ARG: - arg = code2num(code, offset+1) | extended_arg + arg = code2num(code, offset + 1) | extended_arg extended_arg = extended_arg_val(self.opc, arg) continue @@ -425,8 +447,11 @@ class Scanner(object): last_was_extarg = False n = len(instructions) for i, inst in enumerate(instructions): - if (inst.opname == 'EXTENDED_ARG' - and i+1 < n and instructions[i+1].opname != 'MAKE_FUNCTION'): + if ( + inst.opname == "EXTENDED_ARG" + and i + 1 < n + and instructions[i + 1].opname != "MAKE_FUNCTION" + ): last_was_extarg = True starts_line = inst.starts_line is_jump_target = inst.is_jump_target @@ -437,13 +462,15 @@ class Scanner(object): # j = self.stmts.index(inst.offset) # self.lines[j] = offset - new_inst = inst._replace(starts_line=starts_line, - is_jump_target=is_jump_target, - offset=offset) + new_inst = inst._replace( + starts_line=starts_line, + is_jump_target=is_jump_target, + offset=offset, + ) inst = new_inst if i < n: new_prev = self.prev_op[instructions[i].offset] - j = instructions[i+1].offset + j = instructions[i + 1].offset old_prev = self.prev_op[j] while self.prev_op[j] == old_prev and j < n: self.prev_op[j] = new_prev @@ -465,9 +492,12 @@ class Scanner(object): for i in ifs: # For each offset, if line number of current and next op # is the same - if self.lines[i].l_no == self.lines[i+3].l_no: + if self.lines[i].l_no == self.lines[i + 3].l_no: # Skip last op on line if it is some sort of POP_JUMP. - if self.code[self.prev[self.lines[i].next]] in (self.opc.PJIT, self.opc.PJIF): + if self.code[self.prev[self.lines[i].next]] in ( + self.opc.PJIT, + self.opc.PJIF, + ): continue filtered.append(i) return filtered @@ -477,8 +507,8 @@ class Scanner(object): def restrict_to_parent(self, target, parent): """Restrict target to parent structure boundaries.""" - if not (parent['start'] < target < parent['end']): - target = parent['end'] + if not (parent["start"] < target < parent["end"]): + target = parent["end"] return target def setTokenClass(self, tokenClass): @@ -486,6 +516,7 @@ class Scanner(object): self.Token = tokenClass return self.Token + def parse_fn_counts(argc): return ((argc & 0xFF), (argc >> 8) & 0xFF, (argc >> 16) & 0x7FFF) @@ -498,8 +529,10 @@ def get_scanner(version, is_pypy=False, show_asm=None): raise RuntimeError("Unknown Python version in xdis %s" % version) canonic_version = canonic_python_version[version] if canonic_version not in CANONIC2VERSION: - raise RuntimeError("Unsupported Python version %s (canonic %s)" - % (version, canonic_version)) + raise RuntimeError( + "Unsupported Python version %s (canonic %s)" + % (version, canonic_version) + ) version = CANONIC2VERSION[canonic_version] # Pick up appropriate scanner @@ -507,24 +540,34 @@ def get_scanner(version, is_pypy=False, show_asm=None): v_str = "%s" % (int(version * 10)) try: import importlib + if is_pypy: scan = importlib.import_module("uncompyle6.scanners.pypy%s" % v_str) else: scan = importlib.import_module("uncompyle6.scanners.scanner%s" % v_str) - if False: print(scan) # Avoid unused scan + if False: + print(scan) # Avoid unused scan except ImportError: if is_pypy: - exec("import uncompyle6.scanners.pypy%s as scan" % v_str, - locals(), globals()) + exec( + "import uncompyle6.scanners.pypy%s as scan" % v_str, + locals(), + globals(), + ) else: - exec("import uncompyle6.scanners.scanner%s as scan" % v_str, - locals(), globals()) + exec( + "import uncompyle6.scanners.scanner%s as scan" % v_str, + locals(), + globals(), + ) if is_pypy: - scanner = eval("scan.ScannerPyPy%s(show_asm=show_asm)" % v_str, - locals(), globals()) + scanner = eval( + "scan.ScannerPyPy%s(show_asm=show_asm)" % v_str, locals(), globals() + ) else: - scanner = eval("scan.Scanner%s(show_asm=show_asm)" % v_str, - locals(), globals()) + scanner = eval( + "scan.Scanner%s(show_asm=show_asm)" % v_str, locals(), globals() + ) else: raise RuntimeError("Unsupported Python version %s" % version) return scanner @@ -532,8 +575,9 @@ def get_scanner(version, is_pypy=False, show_asm=None): if __name__ == "__main__": import inspect, uncompyle6 + co = inspect.currentframe().f_code # scanner = get_scanner('2.7.13', True) # scanner = get_scanner(sys.version[:5], False) scanner = get_scanner(uncompyle6.PYTHON_VERSION, IS_PYPY, True) - tokens, customize = scanner.ingest(co, {}, show_asm='after') + tokens, customize = scanner.ingest(co, {}, show_asm="after") diff --git a/uncompyle6/scanners/scanner3.py b/uncompyle6/scanners/scanner3.py index 56994193..6c171393 100644 --- a/uncompyle6/scanners/scanner3.py +++ b/uncompyle6/scanners/scanner3.py @@ -268,7 +268,10 @@ class Scanner3(Scanner): # There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n else: - assert_can_follow = inst.opname == "POP_JUMP_IF_TRUE" and i + 1 < n + assert_can_follow = ( + inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE") + and i + 1 < n + ) if assert_can_follow: next_inst = self.insts[i + 1] if ( @@ -278,9 +281,7 @@ class Scanner3(Scanner): ): raise_idx = self.offset2inst_index[self.prev_op[inst.argval]] raise_inst = self.insts[raise_idx] - if raise_inst.opname.startswith( - "RAISE_VARARGS" - ): + if raise_inst.opname.startswith("RAISE_VARARGS"): self.load_asserts.add(next_inst.offset) pass pass @@ -436,11 +437,16 @@ class Scanner3(Scanner): else: opname = "%s_%d" % (opname, pos_args) - elif self.is_pypy and opname == "JUMP_IF_NOT_DEBUG": - # The value in the dict is in special cases in semantic actions, such - # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put - # in arbitrary value 0. - customize[opname] = 0 + elif self.is_pypy and opname in ("JUMP_IF_NOT_DEBUG", "CALL_FUNCTION"): + if opname == "JUMP_IF_NOT_DEBUG": + # The value in the dict is in special cases in semantic actions, such + # as JUMP_IF_NOT_DEBUG. The value is not used in these cases, so we put + # in arbitrary value 0. + customize[opname] = 0 + elif self.version >= 3.6 and argval > 255: + opname = "CALL_FUNCTION_KW" + pass + elif opname == "UNPACK_EX": # FIXME: try with scanner and parser by # changing argval diff --git a/uncompyle6/semantics/customize.py b/uncompyle6/semantics/customize.py index 95ad4b19..fe3c5688 100644 --- a/uncompyle6/semantics/customize.py +++ b/uncompyle6/semantics/customize.py @@ -28,8 +28,15 @@ def customize_for_version(self, is_pypy, version): # PyPy changes ####################### TABLE_DIRECT.update({ - 'assert_pypy': ( '%|assert %c\n' , 1 ), - 'assert2_pypy': ( '%|assert %c, %c\n' , 1, 4 ), + 'assert_pypy': ( '%|assert %c\n' , (1, 'assert_expr') ), + # This is as a result of an if transoration + 'assert0_pypy': ( '%|assert %c\n' , (0, 'assert_expr') ), + + 'assert_not_pypy': ( '%|assert not %c\n' , (1, 'assert_exp') ), + 'assert2_not_pypy': ( '%|assert not %c, %c\n' , (1, 'assert_exp'), + (4, 'expr') ), + 'assert2_pypy': ( '%|assert %c, %c\n' , (1, 'assert_expr'), + (4, 'expr') ), 'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ), 'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ), 'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ), diff --git a/uncompyle6/semantics/customize36.py b/uncompyle6/semantics/customize36.py index f1a04a01..b3a48838 100644 --- a/uncompyle6/semantics/customize36.py +++ b/uncompyle6/semantics/customize36.py @@ -495,7 +495,10 @@ def customize_for_version36(self, version): # bytecode, the escaping of the braces has been # removed. So we need to put back the braces escaping in # reconstructing the source. - assert expr[0] == 'LOAD_STR' + assert ( + expr[0] == "LOAD_STR" or + expr[0] == "LOAD_CONST" and isinstance(expr[0].attr, unicode) + ) value = value.replace("{", "{{").replace("}", "}}") # Remove leading quotes diff --git a/uncompyle6/semantics/fragments.py b/uncompyle6/semantics/fragments.py index 7e1f44d4..5048dc70 100644 --- a/uncompyle6/semantics/fragments.py +++ b/uncompyle6/semantics/fragments.py @@ -189,6 +189,7 @@ class FragmentsWalker(pysource.SourceWalker, object): self.hide_internal = False self.offsets = {} self.last_finish = -1 + self.is_pypy = is_pypy # FIXME: is there a better way? global MAP_DIRECT_FRAGMENT @@ -1463,25 +1464,80 @@ class FragmentsWalker(pysource.SourceWalker, object): # as a custom rule start = len(self.f.getvalue()) n = len(node) - 1 - assert node[n].kind.startswith("CALL_FUNCTION") - for i in range(n - 2, 0, -1): - if not node[i].kind in ["expr", "LOAD_CLASSNAME"]: - break - pass + if node.kind != "expr": + if node == "kwarg": + self.template_engine(("(%[0]{attr}=%c)", 1), node) + return - if i == n - 2: - return - self.write("(") - line_separator = ", " - sep = "" - i += 1 - while i < n: - value = self.traverse(node[i]) - self.node_append(sep, value, node[i]) + kwargs = None + assert node[n].kind.startswith("CALL_FUNCTION") + + if node[n].kind.startswith("CALL_FUNCTION_KW"): + if self.is_pypy: + # FIXME: this doesn't handle positional and keyword args + # properly. Need to do something more like that below + # in the non-PYPY 3.6 case. + self.template_engine(('(%[0]{attr}=%c)', 1), node[n-1]) + return + else: + kwargs = node[n - 1].attr + + assert isinstance(kwargs, tuple) + i = n - (len(kwargs) + 1) + j = 1 + n - node[n].attr + else: + i = start = n - 2 + for i in range(start, 0, -1): + if not node[i].kind in ["expr", "call", "LOAD_CLASSNAME"]: + break + pass + + if i == start: + return + i += 2 + + for i in range(n - 2, 0, -1): + if not node[i].kind in ["expr", "LOAD_CLASSNAME"]: + break + pass + + line_separator = ", " + sep = "" i += 1 - self.write(sep, value) - sep = line_separator + self.write("(") + if kwargs: + # 3.6+ does this + while j < i: + self.write(sep) + value = self.traverse(node[j]) + self.write("%s" % value) + sep = line_separator + j += 1 + + j = 0 + while i < l: + self.write(sep) + value = self.traverse(node[i]) + self.write("%s=%s" % (kwargs[j], value)) + sep = line_separator + j += 1 + i += 1 + else: + while i < l: + value = self.traverse(node[i]) + i += 1 + self.write(sep, value) + sep = line_separator + pass + pass + else: + if self.version >= 3.6 and node[0] == "LOAD_CONST": + return + value = self.traverse(node[0]) + self.write("(") + self.write(value) + pass self.write(")") self.set_pos_info(node, start, len(self.f.getvalue())) diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index b5ad443f..7cc1531c 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -239,7 +239,9 @@ class SourceWalker(GenericASTTraversal, object): is_pypy=is_pypy, ) - self.treeTransform = TreeTransform(version, showast) + self.treeTransform = TreeTransform(version=version, + show_ast=showast, + is_pypy=is_pypy) self.debug_parser = dict(debug_parser) self.showast = showast self.params = params @@ -1564,8 +1566,15 @@ class SourceWalker(GenericASTTraversal, object): assert node[n].kind.startswith("CALL_FUNCTION") if node[n].kind.startswith("CALL_FUNCTION_KW"): - # 3.6+ starts doing this - kwargs = node[n - 1].attr + if self.is_pypy: + # FIXME: this doesn't handle positional and keyword args + # properly. Need to do something more like that below + # in the non-PYPY 3.6 case. + self.template_engine(('(%[0]{attr}=%c)', 1), node[n-1]) + return + else: + kwargs = node[n - 1].attr + assert isinstance(kwargs, tuple) i = n - (len(kwargs) + 1) j = 1 + n - node[n].attr @@ -1750,65 +1759,95 @@ class SourceWalker(GenericASTTraversal, object): else: kv_node = node[1:] else: - assert node[-1].kind.startswith("kvlist") - kv_node = node[-1] - - first_time = True - for kv in kv_node: - assert kv in ("kv", "kv2", "kv3") - - # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR - # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR - # kv3 ::= expr expr STORE_MAP - - # FIXME: DRY this and the above indent = self.indent + " " - if kv == "kv": - self.write(sep) - name = self.traverse(kv[-2], indent="") - if first_time: - line_number = self.indent_if_source_nl(line_number, indent) - first_time = False + line_number = self.line_number + sep = '' + opname = node[-1].kind + if self.is_pypy and self.version >= 3.5: + if opname.startswith('BUILD_CONST_KEY_MAP'): + keys = node[-2].attr + # FIXME: DRY this and the above + for i in range(len(keys)): + key = keys[i] + value = self.traverse(node[i], indent='') + self.write(sep, key, ': ', value) + sep = ", " + if line_number != self.line_number: + sep += "\n" + self.indent + " " + line_number = self.line_number + pass + pass pass - line_number = self.line_number - self.write(name, ": ") - value = self.traverse( - kv[1], indent=self.indent + (len(name) + 2) * " " - ) - elif kv == "kv2": - self.write(sep) - name = self.traverse(kv[1], indent="") - if first_time: - line_number = self.indent_if_source_nl(line_number, indent) - first_time = False + else: + if opname.startswith('kvlist'): + list_node = node[0] + else: + list_node = node + + assert list_node[-1].kind.startswith('BUILD_MAP') + for i in range(0, len(list_node)-1, 2): + key = self.traverse(list_node[i], indent='') + value = self.traverse(list_node[i+1], indent='') + self.write(sep, key, ': ', value) + sep = ", " + if line_number != self.line_number: + sep += "\n" + self.indent + " " + line_number = self.line_number + pass + pass pass - line_number = self.line_number - self.write(name, ": ") - value = self.traverse( - kv[-3], indent=self.indent + (len(name) + 2) * " " - ) - elif kv == "kv3": - self.write(sep) - name = self.traverse(kv[-2], indent="") - if first_time: - line_number = self.indent_if_source_nl(line_number, indent) - first_time = False + elif opname.startswith('kvlist'): + kv_node = node[-1] + first_time = True + for kv in kv_node: + assert kv in ('kv', 'kv2', 'kv3') + + # kv ::= DUP_TOP expr ROT_TWO expr STORE_SUBSCR + # kv2 ::= DUP_TOP expr expr ROT_THREE STORE_SUBSCR + # kv3 ::= expr expr STORE_MAP + + # FIXME: DRY this and the above + if kv == 'kv': + self.write(sep) + name = self.traverse(kv[-2], indent='') + if first_time: + line_number = self.indent_if_source_nl(line_number, indent) + first_time = False + pass + line_number = self.line_number + self.write(name, ': ') + value = self.traverse(kv[1], indent=self.indent+(len(name)+2)*' ') + elif kv == 'kv2': + self.write(sep) + name = self.traverse(kv[1], indent='') + if first_time: + line_number = self.indent_if_source_nl(line_number, indent) + first_time = False + pass + line_number = self.line_number + self.write(name, ': ') + value = self.traverse(kv[-3], indent=self.indent+(len(name)+2)*' ') + elif kv == 'kv3': + self.write(sep) + name = self.traverse(kv[-2], indent='') + if first_time: + line_number = self.indent_if_source_nl(line_number, indent) + first_time = False + pass + line_number = self.line_number + self.write(name, ': ') + line_number = self.line_number + value = self.traverse(kv[0], indent=self.indent+(len(name)+2)*' ') + pass + self.write(value) + sep = ", " + if line_number != self.line_number: + sep += "\n" + self.indent + " " + line_number = self.line_number + pass pass - line_number = self.line_number - self.write(name, ": ") - line_number = self.line_number - value = self.traverse( - kv[0], indent=self.indent + (len(name) + 2) * " " - ) - pass - self.write(value) - sep = ", " - if line_number != self.line_number: - sep += "\n" + self.indent + " " - line_number = self.line_number - pass + pass - pass if sep.startswith(",\n"): self.write(sep[1:]) if node[0] != "dict_entry": diff --git a/uncompyle6/semantics/transform.py b/uncompyle6/semantics/transform.py index a194c964..7156db40 100644 --- a/uncompyle6/semantics/transform.py +++ b/uncompyle6/semantics/transform.py @@ -30,9 +30,11 @@ def is_docstring(node): class TreeTransform(GenericASTTraversal, object): - def __init__(self, version, show_ast=None): + def __init__(self, version, show_ast=None, + is_pypy=False): self.version = version self.showast = show_ast + self.is_pypy = is_pypy return def maybe_show_tree(self, ast): @@ -133,7 +135,10 @@ class TreeTransform(GenericASTTraversal, object): # becomes: # assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM if jump_cond == "jmp_true": - kind = "assert" + if self.is_pypy: + kind = "assert0_pypy" + else: + kind = "assert" else: assert jump_cond == "jmp_false" kind = "assertnot" @@ -230,6 +235,15 @@ class TreeTransform(GenericASTTraversal, object): n_ifelsestmtc = n_ifelsestmtl = n_ifelsestmt + def n_list_for(self, list_for_node): + expr = list_for_node[0] + if (expr == "expr" and expr[0] == "get_iter"): + # Remove extraneous get_iter() inside the "for" of a comprehension + assert expr[0][0] == "expr" + list_for_node[0] = expr[0][0] + list_for_node.transformed_by="n_list_for", + return list_for_node + def traverse(self, node, is_lambda=False): node = self.preorder(node) return node