Merge branch 'master' into python-2.4

NEWS.md

@@ -1,4 +1,18 @@
-3.3.0 2019-03-23 Holy Week
+3.3.1 2019-04-19 Good Friday
+==========================
+
+Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry though, many more remain.
+
+* Add annotation return values in 3.6+
+* Fix 3.6+ lambda parameter handling decompilation
+* Fix 3.7+ chained comparision decompilation
+* split out semantic-action customization into more separate files
+* Add 3.8 try/else
+* Fix 2.7 generator decompilation
+* Fix some parser failures fixes in 3.4+ using test_pyenvlib
+* Add more run tests
+
+3.3.0 2019-43-14 Holy Week
 ==========================
 
 * First cut at Python 3.8 (many bug remain)

admin-tools/pyenv-newer-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.2.6 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'
+export PYVERSIONS='3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'

test/simple_source/bug33/01_triple_compare.py

@@ -1,8 +1,11 @@
 # In Python 3.3+ this uses grammar rule
 # compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
+# In Python 3.6 uses this uses grammar rule
+# compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
 
 # Seen in Python 3.3 ipaddress.py
 
+# RUNNABLE!
 def _is_valid_netmask(netmask):
     return 0 <= netmask <= 10
 
@@ -10,4 +13,4 @@ def _is_valid_netmask(netmask):
 # detections
 
 # See in 2.6.9 quopri.py ishex():
-'0' <= __file__ <= '9' or 'a' <= __file__ <= 'f' or 'A' <= __file__ <= 'F'
+assert not '0' <= __file__ <= '9' or 'a' <= __file__ <= 'f' or 'A' <= __file__ <= 'F'

test/simple_source/bug33/02_pos_args.py

@@ -1,6 +1,12 @@
 # From Python 3.3.6 hmac.py
-# Problem was getting wrong placement of positional args
+# Problem was getting wrong placement of positional args.
+# In 3.6+ paramter handling changes
+
+# RUNNABLE!
+
 digest_cons = lambda d=b'': 5
 
 # Handle single kwarg
-lambda *, d=0: None
+x = lambda *, d=0: d
+assert x(d=1) == 1
+assert x() == 0

test/simple_source/bug36/05_try_whiletrue.py

@@ -0,0 +1,19 @@
+# From 3.6 _collections.abc.py
+# Bug was try/execpt parsing detection since 3.6 removes
+# a JUMP_FORWARD from earlier 3.xs.
+# This could also get confused with try/else.
+
+# RUNNABLE!
+def iter(self):
+    i = 0
+    try:
+        while True:
+            v = self[i]
+            yield v
+            i += 1
+    except IndexError:
+        return
+
+
+A = [10, 20, 30]
+assert list(iter(A)) == A

uncompyle6/parsers/parse3.py

@@ -99,7 +99,7 @@ class Python3Parser(PythonParser):
         sstmt ::= return RETURN_LAST
 
         return_if_stmts ::= return_if_stmt come_from_opt
-        return_if_stmts ::= _stmts return_if_stmt
+        return_if_stmts ::= _stmts return_if_stmt _come_froms
         return_if_stmt  ::= ret_expr RETURN_END_IF
         returns         ::= _stmts return_if_stmt
 
@@ -954,11 +954,17 @@ class Python3Parser(PythonParser):
                                         opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
 
+                    else:
+                        rule = ('mklambda ::= %sLOAD_LAMBDA LOAD_CONST %s' %
+                                (('expr ' * stack_count), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
+
                     rule = ('mkfunc ::= %s%s%s%s' %
-                                ('expr ' * stack_count,
-                                 'load_closure ' * closure,
-                                 'LOAD_CONST ' * 2,
-                                 opname))
+                            ('expr ' * stack_count,
+                             'load_closure ' * closure,
+                             'LOAD_CONST ' * 2,
+                             opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
 
                     if has_get_iter_call_function1:
@@ -1150,7 +1156,9 @@ class Python3Parser(PythonParser):
         self.check_reduce['ifelsestmt'] = 'AST'
         self.check_reduce['annotate_tuple'] = 'noAST'
         self.check_reduce['kwarg'] = 'noAST'
-        self.check_reduce['try_except'] = 'AST'
+        if self.version < 3.6:
+            # 3.6+ can remove a JUMP_FORWARD which messes up our testing here
+            self.check_reduce['try_except'] = 'AST'
 
         # FIXME: remove parser errors caused by the below
         # self.check_reduce['while1elsestmt'] = 'noAST'

uncompyle6/parsers/parse34.py

@@ -17,15 +17,10 @@ spark grammar differences over Python 3.3 for Python 3.4
 """
 
 from uncompyle6.parser import PythonParserSingle
-from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.parsers.parse33 import Python33Parser
 
 class Python34Parser(Python33Parser):
 
-    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
-        super(Python34Parser, self).__init__(debug_parser)
-        self.customized = {}
-
     def p_misc34(self, args):
         """
         expr ::= LOAD_ASSERT

uncompyle6/parsers/parse36.py

@@ -132,6 +132,8 @@ class Python36Parser(Python35Parser):
         stmt ::= tryfinally_return_stmt
         tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
                                    COME_FROM_FINALLY
+
+        compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/parse37.py

@@ -100,9 +100,11 @@ class Python37Parser(Python36Parser):
 
         compare_chained2a_37       ::= expr COMPARE_OP POP_JUMP_IF_TRUE JUMP_FORWARD
         compare_chained2a_false_37 ::= expr COMPARE_OP POP_JUMP_IF_FALSE JUMP_FORWARD
-        compare_chained2b_37       ::= expr COMPARE_OP COME_FROM POP_JUMP_IF_FALSE JUMP_FORWARD ELSE
+
+        compare_chained2b_37       ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD ELSE
+
         compare_chained2c_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
-                                       compare_chained2a_false_37  ELSE
+                                       compare_chained2a_false_37 ELSE
 
         _ifstmts_jump        ::= c_stmts_opt come_froms
         """

uncompyle6/semantics/customize37.py

@@ -46,6 +46,9 @@ def customize_for_version37(self, version):
         'compare_chained2a_37': (
             '%[1]{pattr.replace("-", " ")} %p',
             (0, 19) ),
+        'compare_chained2b_37': (
+            '%[1]{pattr.replace("-", " ")} %p',
+            (0, 19) ),
         'compare_chained2a_false_37': (
             '%[1]{pattr.replace("-", " ")} %p',
             (0, 19 ) ),

uncompyle6/semantics/make_function.py

@@ -507,11 +507,47 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     args_node = node[-1]
 
+    annotate_dict = {}
+
     # Get a list of tree nodes that constitute the values for the "default
     # parameters"; these are default values that appear before any *, and are
     # not to be confused with keyword parameters which may appear after *.
-    if isinstance(args_node.attr, tuple):
-        pos_args, kw_args, annotate_argc  = args_node.attr
+    args_attr = args_node.attr
+
+    if isinstance(args_attr, tuple) or (self.version >= 3.6 and isinstance(args_attr, list)):
+        if len(args_attr) == 3:
+            pos_args, kw_args, annotate_argc  = args_attr
+        else:
+            pos_args, kw_args, annotate_argc, closure  = args_attr
+
+            i = -4
+            kw_pairs = 0
+            if closure:
+                # FIXME: fill in
+                i -= 1
+            if annotate_argc:
+                # Turn into subroutine and DRY with other use
+                annotate_node = node[i]
+                if annotate_node == 'expr':
+                    annotate_node = annotate_node[0]
+                    annotate_name_node = annotate_node[-1]
+                    if annotate_node == 'dict' and annotate_name_node.kind.startswith('BUILD_CONST_KEY_MAP'):
+                        types = [self.traverse(n, indent='') for n in annotate_node[:-2]]
+                        names = annotate_node[-2].attr
+                        l = len(types)
+                        assert l == len(names)
+                        for i in range(l): annotate_dict[names[i]] = types[i]
+                        pass
+                    pass
+                i -= 1
+            if kw_args:
+                kw_node = node[i]
+                if kw_node == 'expr':
+                    kw_node = kw_node[0]
+                if kw_node == 'dict':
+                    kw_pairs = kw_node[-1].attr
+
+
         # FIXME: there is probably a better way to classify this.
         have_kwargs = node[0].kind.startswith('kwarg') or node[0] == 'no_kwargs'
         if len(node) >= 4:
@@ -536,9 +572,26 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 default_values_start += 1
             defparams = node[default_values_start:default_values_start+args_node.attr[0]]
         else:
-            # args are first, before kwargs. Or there simply are no kwargs.
-            defparams = node[:args_node.attr[0]]
-            pass
+            if self.version < 3.6:
+                defparams = node[:args_node.attr[0]]
+                kw_args  = 0
+            else:
+                defparams = []
+                # FIXME: DRY with code below
+                default, kw_args, annotate_argc = args_node.attr[0:3]
+                if default:
+                    expr_node = node[0]
+                    if node[0] == 'pos_arg':
+                        expr_node = expr_node[0]
+                    assert expr_node == 'expr', "expecting mkfunc default node to be an expr"
+                    if (expr_node[0] == 'LOAD_CONST' and
+                        isinstance(expr_node[0].attr, tuple)):
+                        defparams = [repr(a) for a in expr_node[0].attr]
+                    elif expr_node[0] in frozenset(('list', 'tuple', 'dict', 'set')):
+                        defparams =  [self.traverse(n, indent='') for n in expr_node[0][:-1]]
+                else:
+                    defparams = []
+                pass
     else:
         if self.version < 3.6:
             defparams = node[:args_node.attr]
@@ -562,9 +615,22 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             kw_pairs = 0
             if closure:
                 # FIXME: fill in
+                annotate = node[i]
                 i -= 1
-            if annotate:
-                # FIXME: fill in
+            if annotate_argc:
+                # Turn into subroutine and DRY with other use
+                annotate_node = node[i]
+                if annotate_node == 'expr':
+                    annotate_node = annotate_node[0]
+                    annotate_name_node = annotate_node[-1]
+                    if annotate_node == 'dict' and annotate_name_node.kind.startswith('BUILD_CONST_KEY_MAP'):
+                        types = [self.traverse(n, indent='') for n in annotate_node[:-2]]
+                        names = annotate_node[-2].attr
+                        l = len(types)
+                        assert l == len(names)
+                        for i in range(l): annotate_dict[names[i]] = types[i]
+                        pass
+                    pass
                 i -= 1
             if kw_args:
                 kw_node = node[i]
@@ -654,6 +720,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             ast[-1] = ast_expr
             pass
     else:
+        # FIXME: add annotations here
         self.write("(", ", ".join(params))
     # self.println(indent, '#flags:\t', int(code.co_flags))
 
@@ -752,7 +819,10 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     if is_lambda:
         self.write(": ")
     else:
-        self.println("):")
+        self.write(')')
+        if annotate_dict and 'return' in annotate_dict:
+            self.write(' -> %s' % annotate_dict['return'])
+        self.println(":")
 
     if len(code.co_consts) > 0 and code.co_consts[0] is not None and not is_lambda: # ugly
         # docstring exists, dump it

uncompyle6/semantics/pysource.py

@@ -811,6 +811,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.write(func_name)
 
         self.indent_more()
+
         self.make_function(node, is_lambda=False, code_node=code_node)
 
         if len(self.param_stack) > 1:

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.3.0'  # noqa
+VERSION='3.3.1'  # noqa