diff --git a/test/Makefile b/test/Makefile index e463313f..b3d82c9a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -10,10 +10,10 @@ NATIVE_CHECK = check-$(PYTHON_VERSION) COMPILE ?= #: Run working tests from Python 2.7 -check-2.7: check-short-2.7 check-bytecode check-2.7-ok +check-2.7: check-short-2.7 check-bytecode-2.5 check-2.7-ok #: Run working tests from Python 3.4 -check-3.4: check-short-3.4 check-short-2.7 check-bytecode +check-3.4: check-short-3.4 check-short-2.7 check-bytecode-2.5 check-bytecode-3.2 check: check-short @$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \ @@ -27,10 +27,15 @@ check-short: check-disasm: $(PYTHON) dis-compare.py -#: Check deparsing only, but from a different Python version -check-bytecode: +#: Check deparsing only, from Python 2.5 +check-bytecode-2.5: $(PYTHON) test_pythonlib.py --bytecode-2.5 + +#: Check deparsing only, from Python 3.2 +check-bytecode-3.2: + $(PYTHON) test_pythonlib.py --bytecode-3.2 + #: short tests for bytecodes only for this version of Python check-native-short: $(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE) diff --git a/test/bytecode_2.7/01-lc.pyc b/test/bytecode_2.7/01-lc.pyc new file mode 100644 index 00000000..9ac619fd Binary files /dev/null and b/test/bytecode_2.7/01-lc.pyc differ diff --git a/test/bytecode_3.4/01-lc.pyc b/test/bytecode_3.4/01-lc.pyc new file mode 100644 index 00000000..70659be9 Binary files /dev/null and b/test/bytecode_3.4/01-lc.pyc differ diff --git a/test/simple-source/comprehension/01-lc.py b/test/simple-source/comprehension/01-lc.py new file mode 100644 index 00000000..1d896069 --- /dev/null +++ b/test/simple-source/comprehension/01-lc.py @@ -0,0 +1,18 @@ +# Simple list commprehensions +# +# Python2 grammar includes: +# list_compr ::= BUILD_LIST_0 list_iter +# list_iter ::= list_for +# list_for ::= expr _for designator list_iter JUMP_BACK +# list_iter ::= lc_body +# lc_body ::= expr LIST_APPEND +# +# Python3 grammar includes: +# listcomp ::= LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 + +[ i for i in (1, 2, 3, 4) ] +[ i+1 for i in (1, 2, 3, 4) ] + +# Try next: +# [ i * i for i in range(4) ] +# [ i * j for i in range(4) for j in range(7) ] diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index 64a89582..0453c57d 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -63,6 +63,9 @@ test_options = { 'bytecode-2.7': ['bytecode_2.7', PYC, 'bytecode_2.7', 2.7], + 'bytecode-3.2': + ['bytecode_3.2', PYC, 'bytecode_3.2', 3.2], + 'bytecode-3.4': ['bytecode_3.4', PYC, 'bytecode_3.4', 3.4], diff --git a/uncompyle6/deparser.py b/uncompyle6/deparser.py index 4157e81e..42ba3c85 100644 --- a/uncompyle6/deparser.py +++ b/uncompyle6/deparser.py @@ -483,7 +483,7 @@ class Traverser(walker.Walker, object): n = ast[iter_index] assert n == 'comp_iter' - # find innerst node + # find innermost node while n == 'comp_iter': n = n[0] # recurse one step if n == 'comp_for': n = n[3] @@ -494,14 +494,66 @@ class Traverser(walker.Walker, object): self.preorder(n[0]) self.write(' for ') start = len(self.f.getvalue()) - self.preorder(ast[iter_index-1]) - self.set_pos_info(node, start, len(self.f.getvalue())) + designator = ast[iter_index-1] + self.preorder(designator) + self.set_pos_info(ast[iter_index-1], start, len(self.f.getvalue())) self.write(' in ') start = len(self.f.getvalue()) node[-3].parent = node self.preorder(node[-3]) - self.set_pos_info(node, start, len(self.f.getvalue())) + self.set_pos_info(node[-3], start, len(self.f.getvalue())) + start = len(self.f.getvalue()) self.preorder(ast[iter_index]) + self.set_pos_info(iter_index, start, len(self.f.getvalue())) + self.prec = p + + def listcomprehension_walk3(self, node, iter_index, code_index=-5): + """List comprehensions the way they are done in Python3. + They're more other comprehensions, e.g. set comprehensions + See if we can combine code. + """ + p = self.prec + self.prec = 27 + code = node[code_index].attr + + assert inspect.iscode(code) + code = Code(code, self.scanner, self.currentclass) + # assert isinstance(code, Code) + + ast = self.build_ast(code._tokens, code._customize) + self.customize(code._customize) + ast = ast[0][0][0][0][0] + + n = ast[iter_index] + assert n == 'list_iter' + # find innermost node + while n == 'list_iter': # list_iter + n = n[0] # recurse one step + if n == 'list_for': + designator = n[2] + n = n[3] + elif n == 'list_if': + # FIXME: just a guess + designator = n[1] + + n = n[2] + elif n == 'list_ifnot': + # FIXME: just a guess + designator = n[1] + n = n[2] + assert n == 'lc_body', ast + + self.preorder(n[0]) + self.write(' for ') + start = len(self.f.getvalue()) + self.preorder(designator) + self.set_pos_info(designator, start, len(self.f.getvalue())) + self.write(' in ') + start = len(self.f.getvalue()) + node[-3].parent = node + self.preorder(node[-3]) + self.set_pos_info(node[-3], start, len(self.f.getvalue())) + # self.preorder(ast[iter_index]) self.prec = p def n_genexpr(self, node): diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index f341c77c..af59a959 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -54,6 +54,10 @@ class Python3Parser(PythonParser): def p_list_comprehension(self, args): ''' + expr ::= listcomp + listcomp ::= LOAD_LISTCOMP LOAD_CONST MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 + + expr ::= list_compr list_compr ::= BUILD_LIST_0 list_iter @@ -65,7 +69,7 @@ class Python3Parser(PythonParser): _come_from ::= COME_FROM _come_from ::= - list_for ::= expr _for designator list_iter JUMP_BACK + list_for ::= expr FOR_ITER designator list_iter JUMP_ABSOLUTE list_if ::= expr jmp_false list_iter list_if_not ::= expr jmp_true list_iter @@ -96,7 +100,7 @@ class Python3Parser(PythonParser): comp_if ::= expr jmp_false comp_iter comp_ifnot ::= expr jmp_true comp_iter - comp_for ::= expr _for designator comp_iter JUMP_BACK + comp_for ::= expr _for designator comp_iter JUMP_ABSOLUTE ''' def p_genexpr(self, args): diff --git a/uncompyle6/scanners/scanner34.py b/uncompyle6/scanners/scanner34.py index e720bdff..a9aace45 100644 --- a/uncompyle6/scanners/scanner34.py +++ b/uncompyle6/scanners/scanner34.py @@ -15,7 +15,7 @@ for later use in deparsing. from __future__ import print_function -import dis +import dis, inspect from collections import namedtuple from array import array @@ -108,7 +108,27 @@ class Scanner34(scan.Scanner): # keyword attribute names in a call. I suspect there will be things # other than LOAD_CONST, but we'll start out with just this for now. if opname in ['LOAD_CONST']: - pattr = inst.argval + const = inst.argval + if inspect.iscode(const): + if const.co_name == '': + opname = 'LOAD_LAMBDA' + elif const.co_name == '': + opname = 'LOAD_GENEXPR' + elif const.co_name == '': + opname = 'LOAD_DICTCOMP' + elif const.co_name == '': + opname = 'LOAD_SETCOMP' + elif const.co_name == '': + opname = 'LOAD_LISTCOMP' + # verify() uses 'pattr' for comparison, since 'attr' + # now holds Code(const) and thus can not be used + # for comparison (todo: think about changing this) + # pattr = 'code_object @ 0x%x %s->%s' %\ + # (id(const), const.co_filename, const.co_name) + pattr = '' + else: + pattr = const + pass elif opname in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET', 'BUILD_SLICE', 'UNPACK_SEQUENCE', 'MAKE_FUNCTION', 'MAKE_CLOSURE', diff --git a/uncompyle6/walker.py b/uncompyle6/walker.py index fc2bbae4..e430d80e 100644 --- a/uncompyle6/walker.py +++ b/uncompyle6/walker.py @@ -910,9 +910,12 @@ class Walker(GenericASTTraversal, object): self.prune() # stop recursing def n_list_compr(self, node): + """List comprehensions the way they are done in Python2. + """ p = self.prec self.prec = 27 n = node[-1] + assert n == 'list_iter' # find innerst node while n == 'list_iter': @@ -928,10 +931,10 @@ class Walker(GenericASTTraversal, object): self.prec = p self.prune() # stop recursing - def comprehension_walk(self, node, iter_index): + def comprehension_walk(self, node, iter_index, code_index=-5): p = self.prec self.prec = 27 - code = node[-5].attr + code = node[code_index].attr assert inspect.iscode(code) code = Code(code, self.scanner, self.currentclass) @@ -940,11 +943,10 @@ class Walker(GenericASTTraversal, object): ast = self.build_ast(code._tokens, code._customize) self.customize(code._customize) ast = ast[0][0][0] - n = ast[iter_index] assert n == 'comp_iter' # find innerst node - while n == 'comp_iter': + while n == 'comp_iter': # list_iter n = n[0] # recurse one step if n == 'comp_for': n = n[3] elif n == 'comp_if': n = n[2] @@ -961,16 +963,66 @@ class Walker(GenericASTTraversal, object): def n_genexpr(self, node): self.write('(') - self.comprehension_walk(node, 3) + self.comprehension_walk(node, iter_index=3) self.write(')') self.prune() def n_setcomp(self, node): self.write('{') - self.comprehension_walk(node, 4) + self.comprehension_walk(node, iter_index=4) self.write('}') self.prune() + def listcomprehension_walk3(self, node, iter_index, code_index=-5): + """List comprehensions the way they are done in Python3. + They're more other comprehensions, e.g. set comprehensions + See if we can combine code. + """ + p = self.prec + self.prec = 27 + code = node[code_index].attr + + assert inspect.iscode(code) + code = Code(code, self.scanner, self.currentclass) + # assert isinstance(code, Code) + + ast = self.build_ast(code._tokens, code._customize) + self.customize(code._customize) + ast = ast[0][0][0][0][0] + + n = ast[iter_index] + assert n == 'list_iter' + # find innermost node + while n == 'list_iter': # list_iter + n = n[0] # recurse one step + if n == 'list_for': + designator = n[2] + n = n[3] + elif n == 'list_if': + # FIXME: just a guess + designator = n[1] + + n = n[2] + elif n == 'list_ifnot': + # FIXME: just a guess + designator = n[1] + n = n[2] + assert n == 'lc_body', ast + + self.preorder(n[0]) + self.write(' for ') + self.preorder(designator) + self.write(' in ') + self.preorder(node[-3]) + # self.preorder(ast[iter_index]) + self.prec = p + + def n_listcomp(self, node): + self.write('[') + self.listcomprehension_walk3(node, iter_index=1, code_index=0) + self.write(']') + self.prune() + n_dictcomp = n_setcomp def n_classdef(self, node):