diff --git a/test/bytecode_2.6/04_withas.pyc b/test/bytecode_2.6/04_withas.pyc index 56344384..bfb6e394 100644 Binary files a/test/bytecode_2.6/04_withas.pyc and b/test/bytecode_2.6/04_withas.pyc differ diff --git a/test/bytecode_2.6/10_lambda.pyc b/test/bytecode_2.6/10_lambda.pyc index 63e2a408..b64c4834 100644 Binary files a/test/bytecode_2.6/10_lambda.pyc and b/test/bytecode_2.6/10_lambda.pyc differ diff --git a/test/bytecode_2.7/01_float.pyc b/test/bytecode_2.7/01_float.pyc new file mode 100644 index 00000000..db9f8c70 Binary files /dev/null and b/test/bytecode_2.7/01_float.pyc differ diff --git a/test/bytecode_2.7/01_triple_compare.pyc b/test/bytecode_2.7/01_triple_compare.pyc new file mode 100644 index 00000000..d3bd01c3 Binary files /dev/null and b/test/bytecode_2.7/01_triple_compare.pyc differ diff --git a/test/bytecode_2.7/03_tuple_assign.pyc b/test/bytecode_2.7/03_tuple_assign.pyc index 01b13823..091b86ba 100644 Binary files a/test/bytecode_2.7/03_tuple_assign.pyc and b/test/bytecode_2.7/03_tuple_assign.pyc differ diff --git a/test/bytecode_2.7/04_withas.pyc b/test/bytecode_2.7/04_withas.pyc index 4223a885..61acd311 100644 Binary files a/test/bytecode_2.7/04_withas.pyc and b/test/bytecode_2.7/04_withas.pyc differ diff --git a/test/bytecode_2.7/10_lambda.pyc b/test/bytecode_2.7/10_lambda.pyc new file mode 100644 index 00000000..b22e79ec Binary files /dev/null and b/test/bytecode_2.7/10_lambda.pyc differ diff --git a/test/bytecode_3.0/04_withas.pyc b/test/bytecode_3.0/04_withas.pyc index 7daa1c3a..f0574c2e 100644 Binary files a/test/bytecode_3.0/04_withas.pyc and b/test/bytecode_3.0/04_withas.pyc differ diff --git a/test/bytecode_3.1/04_withas.pyc b/test/bytecode_3.1/04_withas.pyc index d9c2af12..99ad4503 100644 Binary files a/test/bytecode_3.1/04_withas.pyc and b/test/bytecode_3.1/04_withas.pyc differ diff --git a/test/bytecode_3.2/10_lambda.pyc b/test/bytecode_3.2/10_lambda.pyc index 32a80130..969ace01 100644 Binary files a/test/bytecode_3.2/10_lambda.pyc and b/test/bytecode_3.2/10_lambda.pyc differ diff --git a/test/bytecode_3.3/04_withas.pyc b/test/bytecode_3.3/04_withas.pyc index e1b54e5a..841c708a 100644 Binary files a/test/bytecode_3.3/04_withas.pyc and b/test/bytecode_3.3/04_withas.pyc differ diff --git a/test/simple_source/bug33/01_triple_compare.py b/test/simple_source/bug33/01_triple_compare.py index 5523c6b9..f0985f19 100644 --- a/test/simple_source/bug33/01_triple_compare.py +++ b/test/simple_source/bug33/01_triple_compare.py @@ -1,4 +1,5 @@ # In Python 3.3+ this uses grammar rule -# cmp_list2 ::= expr COMPARE_OP RETURN_VALUE +# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE + def _is_valid_netmask(self, netmask): return 0 <= netmask <= self._max_prefixlen diff --git a/test/simple_source/expression/01_float.py b/test/simple_source/expression/01_float.py new file mode 100644 index 00000000..93a93975 --- /dev/null +++ b/test/simple_source/expression/01_float.py @@ -0,0 +1,4 @@ +a = 1e300 * 1e300 * 0 +b = -1e300 * 1e300 * 0 +c = 1e300 * 1e300 +d = -1e300 * 1e300 diff --git a/test/simple_source/expression/03_tuple_assign.py b/test/simple_source/expression/03_tuple_assign.py index c89a9173..bc7db24c 100644 --- a/test/simple_source/expression/03_tuple_assign.py +++ b/test/simple_source/expression/03_tuple_assign.py @@ -6,7 +6,12 @@ def some_other_function(): some_variable, = some_function() print(some_variable) +# From 2.7 test_compile.py +# Bug is adding erroneous parens in d[(1:2, 1:2)] += 1 +def bug(d): + d[1:2, 1:2] += 1 + empty_tup = () one_item_tup = ("item1", ) -one_item_tup_without_parentheses = "item", +one_item_tup_without_parentheses = "item", many_items_tup = ("item1", "item2", "item3") diff --git a/test/simple_source/expression/10_lambda.py b/test/simple_source/expression/10_lambda.py index 48aa3c90..9631fff5 100644 --- a/test/simple_source/expression/10_lambda.py +++ b/test/simple_source/expression/10_lambda.py @@ -26,3 +26,11 @@ class ExtendedInterpolation(): value_getter = lambda option: self._interpolation.before_get(self, section, option, d[option], d) return value_getter + +# Bug from Python 2.7's test_collections.py +# is that the lambda function has two +# statements in it, one for returning *after* the yield +# The return None statement should be removed and the +# yield should be turned into a statement +def test_Iterable(self): + return (lambda: (yield))() diff --git a/test/simple_source/stmts/04_withas.py b/test/simple_source/stmts/04_withas.py index 3ef0be5f..3ce7bad5 100644 --- a/test/simple_source/stmts/04_withas.py +++ b/test/simple_source/stmts/04_withas.py @@ -10,3 +10,9 @@ def formatweekday(self): with self as encoding: return encoding + +# Bug in 2.7.14 test_contextlib.py. Bug was not enclosing (x,y) in parenthesis +def withas_bug(self, nested, a, b): + with self.assertRaises(ZeroDivisionError): + with nested(a(), b()) as (x, y): + 1 // 0 diff --git a/test/stdlib/README.md b/test/stdlib/README.md new file mode 100644 index 00000000..cc45d1fa --- /dev/null +++ b/test/stdlib/README.md @@ -0,0 +1,3 @@ +The programs in here are to test Python stdlib tests in lib/pythonX.Y/test + +We'll compile a test, then decompile it and then run the test diff --git a/test/stdlib/compile-file.py b/test/stdlib/compile-file.py new file mode 100755 index 00000000..93b37688 --- /dev/null +++ b/test/stdlib/compile-file.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +import sys +if len(sys.argv) != 2: + print("Usage: compile-file.py *python-file*") + sys.exit(1) +source = sys.argv[1] + +assert source.endswith('.py') +basename = source[:-3] + +# We do this crazy way to support Python 2.6 which +# doesn't support version_major, and has a bug in +# floating point so we can't divide 26 by 10 and get +# 2.6 +PY_VERSION = sys.version_info[0] + (sys.version_info[1] / 10.0) + +bytecode = "%s-%s.pyc" % (basename, PY_VERSION) + +import py_compile +print("compiling %s to %s" % (source, bytecode)) +py_compile.compile(source, bytecode, 'exec') +# import os +# os.system("../bin/uncompyle6 %s" % bytecode) diff --git a/test/stdlib/runtests.sh b/test/stdlib/runtests.sh new file mode 100755 index 00000000..8ba94587 --- /dev/null +++ b/test/stdlib/runtests.sh @@ -0,0 +1,58 @@ +#!/bin/bash +me=${BASH_SOURCE[0]} + +# Python version setup +FULLVERSION=${1:-2.7.14} +PYVERSION=${FULLVERSION%.*} +MINOR=${FULLVERSION##?.?.} + +typeset -i STOP_ONERROR=1 +typeset -A SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1) + +# Test directory setup +srcdir=$(dirname $me) +cd $srcdir +fulldir=$(pwd) +TESTDIR=/tmp/test${PYVERSION} +if [[ -e $TESTDIR ]] ; then + rm -fr $TESTDIR +fi +mkdir $TESTDIR || exit $? +cp -r ~/.pyenv/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR +cd $TESTDIR/test +export PYTHONPATH=$TESTDIR + +# Run tests +typeset -i i=0 +typeset -i allerrs=0 +for file in test_*.py; do + [[ -v SKIP_TESTS[$file] ]] && continue + + # If the fails *before* decompiling, skip it! + if ! python $file >/dev/null 2>&1 ; then + continue + fi + + ((i++)) + # (( i > 40 )) && break + short_name=$(basename $file .py) + decompiled_file=$short_name-${PYVERSION}.pyc + $fulldir/compile-file.py $file && \ + mv $file{,.orig} && \ + $fulldir/../../bin/uncompyle6 $decompiled_file > $file + rc=$? + if (( rc == 0 )) ; then + echo ========== Running $file =========== + python $file + rc=$? + else + echo ======= Skipping $file due to compile/decompile errors ======== + fi + (( rc != 0 && allerrs++ )) + if (( STOP_ONERROR && rc )) ; then + echo "** Ran $i tests before failure **" + exit $allerrs + fi +done +echo "Ran $i tests" +exit $allerrs diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index ebffa07a..bfae92bb 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -442,7 +442,7 @@ class PythonParser(GenericASTBuilder): expr ::= load_attr expr ::= binary_expr expr ::= build_list - expr ::= cmp + expr ::= compare expr ::= mapexpr expr ::= and expr ::= or @@ -508,14 +508,17 @@ class PythonParser(GenericASTBuilder): return_lambda ::= ret_expr RETURN_VALUE_LAMBDA LAMBDA_MARKER return_lambda ::= ret_expr RETURN_VALUE_LAMBDA - # Doesn't seemt to be used anymore, but other conditional_lambda's are + # Doesn't seem to be used anymore, but other conditional_lambda's are # conditional_lambda ::= expr jmp_false return_if_stmt return_stmt LAMBDA_MARKER - cmp ::= cmp_list - cmp ::= compare - compare ::= expr expr COMPARE_OP - cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP _come_from - cmp_list2 ::= expr COMPARE_OP JUMP_FORWARD + compare ::= compare_chained + compare ::= compare_single + compare_single ::= expr expr COMPARE_OP + + # A compare_chained is two comparisions like x <= y <= z + compare_chained ::= expr compare_chained1 ROT_TWO POP_TOP _come_from + compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD + mapexpr ::= BUILD_MAP kvlist kvlist ::= kvlist kv diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index e43ef03b..3a25235c 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -201,6 +201,10 @@ class Python2Parser(PythonParser): binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR conditional ::= expr jmp_false expr JUMP_ABSOLUTE expr + + # compare_chained2 is used in a "chained_compare": x <= y <= z + compare_chained2 ::= expr COMPARE_OP RETURN_VALUE + compare_chained2 ::= expr COMPARE_OP RETURN_VALUE_LAMBDA """ def p_slice2(self, args): diff --git a/uncompyle6/parsers/parse26.py b/uncompyle6/parsers/parse26.py index 312bc589..50f0f3f8 100644 --- a/uncompyle6/parsers/parse26.py +++ b/uncompyle6/parsers/parse26.py @@ -243,10 +243,12 @@ class Python26Parser(Python2Parser): def p_misc26(self, args): """ conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt - and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP - cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from - cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list1 _come_from - cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false cmp_list2 _come_from + and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP + + # compare_chained is like x <= y <= z + compare_chained ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_from + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained1 _come_from + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP jmp_false compare_chained2 _come_from return_if_lambda ::= RETURN_END_IF_LAMBDA POP_TOP conditional_lambda ::= expr jmp_false_then expr return_if_lambda diff --git a/uncompyle6/parsers/parse27.py b/uncompyle6/parsers/parse27.py index e877eb54..909a8c2d 100644 --- a/uncompyle6/parsers/parse27.py +++ b/uncompyle6/parsers/parse27.py @@ -77,12 +77,11 @@ class Python27Parser(Python2Parser): or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM - cmp_list1 ::= expr DUP_TOP ROT_THREE - COMPARE_OP JUMP_IF_FALSE_OR_POP - cmp_list1 COME_FROM - cmp_list1 ::= expr DUP_TOP ROT_THREE - COMPARE_OP JUMP_IF_FALSE_OR_POP - cmp_list2 COME_FROM + # compare_chained1 is used exclusively in chained_compare + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP + compare_chained1 COME_FROM + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP + compare_chained2 COME_FROM """ def p_stmt27(self, args): diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index b2df4d05..60b19933 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -332,10 +332,11 @@ class Python3Parser(PythonParser): or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM - cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP - cmp_list1 COME_FROM - cmp_list1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP - cmp_list2 COME_FROM + # compare_chained1 is used exclusively in chained_compare + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP + compare_chained1 COME_FROM + compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP + compare_chained2 COME_FROM """ def p_stmt3(self, args): diff --git a/uncompyle6/parsers/parse32.py b/uncompyle6/parsers/parse32.py index aee72533..99b32c8d 100644 --- a/uncompyle6/parsers/parse32.py +++ b/uncompyle6/parsers/parse32.py @@ -8,8 +8,9 @@ from uncompyle6.parsers.parse3 import Python3Parser class Python32Parser(Python3Parser): def p_32to35(self, args): """ - conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM - cmp_list2 ::= expr COMPARE_OP RETURN_VALUE + conditional ::= expr jmp_false expr jump_forward_else expr COME_FROM + # used exclusively in compare_chained + compare_chained2 ::= expr COMPARE_OP RETURN_VALUE # Store locals is only in Python 3.0 to 3.3 stmt ::= store_locals @@ -55,7 +56,7 @@ class Python32Parser(Python3Parser): def add_custom_rules(self, tokens, customize): # self.remove_rules(""" - # cmp_list2 ::= expr COMPARE_OP RETURN_VALUE + # compare_chained2 ::= expr COMPARE_OP RETURN_VALUE # """) super(Python32Parser, self).add_custom_rules(tokens, customize) for i, token in enumerate(tokens): diff --git a/uncompyle6/semantics/consts.py b/uncompyle6/semantics/consts.py index e9fc7274..e5a3fa9c 100644 --- a/uncompyle6/semantics/consts.py +++ b/uncompyle6/semantics/consts.py @@ -140,9 +140,9 @@ TABLE_DIRECT = { 'binary_subscr': ( '%c[%p]', (0, 'expr'), (1, 100) ), - 'binary_subscr2': ( '%c[%p]', + 'binary_subscr2': ( '%c[%c]', (0, 'expr'), - (1, 100) ), + (1, 'expr') ), 'store_subscr': ( '%c[%c]', 0, 1), 'STORE_FAST': ( '%{pattr}', ), 'STORE_NAME': ( '%{pattr}', ), @@ -192,10 +192,10 @@ TABLE_DIRECT = { 'ret_cond_not': ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ), 'conditional_lambda': ( '%c if %c else %c', 2, 0, 4), - 'compare': ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ), - 'cmp_list': ( '%p %p', (0, 29), (1, 30)), - 'cmp_list1': ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)), - 'cmp_list2': ( '%[1]{pattr} %p', (0, 19)), + 'compare_single': ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ), + 'compare_chained': ( '%p %p', (0, 29), (1, 30)), + 'compare_chained1': ( '%[3]{pattr} %p %p', (0, 19), (-2, 19)), + 'compare_chained2': ( '%[1]{pattr} %p', (0, 19)), # 'classdef': (), # handled by n_classdef() 'funcdef': ( '\n\n%|def %c\n', -2), # -2 to handle closures 'funcdefdeco': ( '\n\n%c', 0), @@ -340,7 +340,7 @@ PRECEDENCE = { 'BINARY_OR': 18, - 'cmp': 20, + 'compare': 20, 'unary_not': 22, diff --git a/uncompyle6/semantics/make_function.py b/uncompyle6/semantics/make_function.py index 69760768..05555111 100644 --- a/uncompyle6/semantics/make_function.py +++ b/uncompyle6/semantics/make_function.py @@ -329,7 +329,6 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): else: defparams = node[:args_node.attr] kw_args = 0 - annotate_argc = 0 pass lambda_index = None @@ -361,10 +360,14 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): self.ERROR = p return +<<<<<<< HEAD if self.version >= 3.0: kw_pairs = args_node.attr[1] else: kw_pairs = 0 +======= + kw_pairs = 0 +>>>>>>> master indent = self.indent # build parameters @@ -380,6 +383,22 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): # dump parameter list (with default values) if isLambda: self.write("lambda ", ", ".join(params)) + # If the last statement is None (which is the + # same thing as "return None" in a lambda) and the + # next to last statement is a "yield". Then we want to + # drop the (return) None since that was just put there + # to have something to after the yield finishes. + # FIXME: this is a bit hoaky and not general + if (len(ast) > 1 and + self.traverse(ast[-1]) == 'None' and + self.traverse(ast[-2]).strip().startswith('yield')): + del ast[-1] + # Now pick out the expr part of the last statement + ast_expr = ast[-1] + while ast_expr.kind != 'expr': + ast_expr = ast_expr[0] + ast[-1] = ast_expr + pass else: self.write("(", ", ".join(params)) @@ -417,7 +436,8 @@ def make_function2(self, node, isLambda, nested=1, codeNode=None): print_docstring(self, indent, code.co_consts[0]) code._tokens = None # save memory - assert ast == 'stmts' + if not isLambda: + assert ast == 'stmts' all_globals = find_all_globals(ast, set()) for g in ((all_globals & self.mod_globs) | find_globals(ast, set())): @@ -540,6 +560,25 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None): self.write("(", ", ".join(params)) # self.println(indent, '#flags:\t', int(code.co_flags)) + # dump parameter list (with default values) + if isLambda: + self.write("lambda ", ", ".join(params)) + # If the last statement is None (which is the + # same thing as "return None" in a lambda) and the + # next to last statement is a "yield". Then we want to + # drop the (return) None since that was just put there + # to have something to after the yield finishes. + # FIXME: this is a bit hoaky and not general + if (len(ast) > 1 and + self.traverse(ast[-1]) == 'None' and + self.traverse(ast[-2]).strip().startswith('yield')): + del ast[-1] + # Now pick out the expr part of the last statement + ast_expr = ast[-1] + while ast_expr.kind != 'expr': + ast_expr = ast_expr[0] + ast[-1] = ast_expr + pass else: if isLambda: self.write("lambda ") diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 08f7b409..572eac2d 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -313,8 +313,10 @@ class SourceWalker(GenericASTTraversal, object): 'importmultiple': ( '%|import %c%c\n', 2, 3 ), 'import_cont' : ( ', %c', 2 ), # With/as is allowed as "from future" thing in 2.5 + # Note: It is safe to put the variables after "as" in parenthesis, + # and sometimes it is needed. 'withstmt': ( '%|with %c:\n%+%c%-', 0, 3), - 'withasstmt': ( '%|with %c as %c:\n%+%c%-', 0, 2, 3), + 'withasstmt': ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3), }) ######################################## @@ -804,7 +806,12 @@ class SourceWalker(GenericASTTraversal, object): def n_LOAD_CONST(self, node): data = node.pattr; datatype = type(data) - if isinstance(datatype, int) and data == minint: + if isinstance(data, float) and str(data) in frozenset(['nan', '-nan', 'inf', '-inf']): + # float values 'nan' and 'inf' are not directly representable in Python at least + # before 3.5 and even there it is via a library constant. + # So we will canonicalize their representation as float('nan') and float('inf') + self.write("float('%s')" % data) + elif isinstance(datatype, int) and data == minint: # convert to hex, since decimal representation # would result in 'LOAD_CONST; UNARY_NEGATIVE' # change:hG/2002-02-07: this was done for all negative integers @@ -1484,6 +1491,11 @@ class SourceWalker(GenericASTTraversal, object): if hasattr(buildclass[-3][0], 'attr'): subclass_code = buildclass[-3][0].attr class_name = buildclass[0].pattr + elif (buildclass[-3] == 'mkfunc' and + node == 'classdefdeco2' and + buildclass[-3][0] == 'load_closure'): + subclass_code = buildclass[-3][1].attr + class_name = buildclass[-3][0][0].pattr elif hasattr(node[0][0], 'pattr'): subclass_code = buildclass[-3][1].attr class_name = node[0][0].pattr @@ -1745,7 +1757,21 @@ class SourceWalker(GenericASTTraversal, object): if lastnodetype.startswith('BUILD_LIST'): self.write('['); endchar = ']' elif lastnodetype.startswith('BUILD_TUPLE'): - self.write('('); endchar = ')' + # Tuples can appear places that can NOT + # have parenthesis around them, like array + # subscripts. We check for that by seeing + # if a tuple item is some sort of slice. + no_parens = False + for n in node: + if n == 'expr' and n[0].kind.startswith('buildslice'): + no_parens = True + break + pass + if no_parens: + endchar = '' + else: + self.write('('); endchar = ')' + pass elif lastnodetype.startswith('BUILD_SET'): self.write('{'); endchar = '}' elif lastnodetype.startswith('BUILD_MAP_UNPACK'):