Merge branch 'master' into python-2.4

NEWS.md

@@ -1,21 +1,36 @@
+3.3.4 2019-05-19 Fleetwood at 65
+================================
+
+Most of the work in this is release is thanks to x0ret.
+
+- Major work was done by x0ret to correct function signatures and include annotation types
+- Handle Python 3.6 STORE_ANNOTATION [#58](https://github.com/rocky/python-uncompyle6/issues/58)
+- Friendlier assembly output
+- `LOAD_CONST` replaced by `LOAD_STR` where appropriate to simplify parsing and improve clarity
+- remove unneeded parenthesis in a generator expression when it is the single argument to the function [#247](https://github.com/rocky/python-uncompyle6/issues/246)
+- Bug in noting an async function [#246](https://github.com/rocky/python-uncompyle6/issues/246)
+- Handle unicode docstrings and fix docstring bugs [#241](https://github.com/rocky/python-uncompyle6/issues/241)
+- Add short option -T as an alternate for --tree+
+- Some grammar cleanup
+
 3.3.3 2019-05-19 Henry and Lewis
 ================================
 
 As before, decomplation bugs fixed. The focus has primarily been on
 Python 3.7. But with this release, releases will be put on hold,as a
-better control-flow detection is worked on . Tis has been needed for a
+better control-flow detection is worked on . This has been needed for a
 while, and is long overdue. It will probably also take a while to get
 done as good as what we have now.
 
 However this work will be done in a new project
 [decompyle3](https://github.com/rocky/python-decompile3).  In contrast
-to _uncompyle6_ the code wil be written assuming a modern Python 3,
+to _uncompyle6_ the code will be written assuming a modern Python 3,
 e.g. 3.7. It is originally intended to decompile Python version 3.7
 and greater.
 
 * A number of Python 3.7+ chained comparisons were fixed
 * Revise Python 3.6ish format string handling
-* Go over operator precedence, e.g. for AST IfExp
+* Go over operator precedence, e.g. for AST `IfExp`
 
 Reported Bug Fixes
 ------------------
@@ -47,7 +62,7 @@ Lots of decomplation bugs, especially in the 3.x series fixed. Don't worry thoug
 
 * Add annotation return values in 3.6+
 * Fix 3.6+ lambda parameter handling decompilation
-* Fix 3.7+ chained comparision decompilation
+* Fix 3.7+ chained comparison decompilation
 * split out semantic-action customization into more separate files
 * Add 3.8 try/else
 * Fix 2.7 generator decompilation
@@ -79,14 +94,14 @@ Bug Fixes
 Pull Requests
 ----------------
 
-* [#202: Better "assert" statement detemination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
+* [#202: Better "assert" statement determination in Python 2.7](https://github.com/rocky/python-uncompyle6/pull/211)
 * [#204: Python 3.7 testing](https://github.com/rocky/python-uncompyle6/pull/204)
 * [#205: Run more f-string tests on Python 3.7](https://github.com/rocky/python-uncompyle6/pull/205)
 * [#211: support utf-8 chars in Python 3 sourcecode](https://github.com/rocky/python-uncompyle6/pull/202)
 
 
 
-3.2.5 2018-12-30 Clearout sale
+3.2.5 2018-12-30 Clear-out sale
 ======================================
 
 - 3.7.2 Remove deprecation warning on regexp string that isn't raw
@@ -151,14 +166,14 @@ Jesus on Friday's New York Times puzzle: "I'm stuck on 2A"
 - reduce 3.5, 3.6 control-flow bugs
 - reduce ambiguity in rules that lead to long (exponential?) parses
 - limit/isolate some 2.6/2.7,3.x grammar rules
-- more runtime testing of decompiled code
+- more run-time testing of decompiled code
-- more removal of parenthesis around calls via setting precidence
+- more removal of parenthesis around calls via setting precedence
 
 3.1.0 2018-03-21 Equinox
 ==============================
 
 - Add code_deparse_with_offset() fragment function.
-- Correct paramenter call fragment deparse_code()
+- Correct parameter call fragment deparse_code()
 - Lots of 3.6, 3.x, and 2.7 bug fixes
   About 5% of 3.6 fail parsing now. But
   semantics still needs much to be desired.

__pkginfo__.py

@@ -58,7 +58,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.7, < 1.9.0',
-                      'xdis >= 4.0.1, < 4.1.0']
+                      'xdis >= 4.0.2, < 4.1.0']
 
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'

pytest/test_grammar.py

@@ -88,7 +88,7 @@ def test_grammar():
             COME_FROM_EXCEPT_CLAUSE
             COME_FROM_LOOP COME_FROM_WITH
             COME_FROM_FINALLY ELSE
-            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+            LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_STR
             LAMBDA_MARKER
             RETURN_END_IF RETURN_END_IF_LAMBDA RETURN_VALUE_LAMBDA RETURN_LAST
             """.split())

pytest/test_token.py

@@ -1,3 +1,4 @@
+from uncompyle6 import PYTHON_VERSION
 from uncompyle6.scanners.tok import Token
 
 def test_token():
@@ -16,7 +17,7 @@ def test_token():
     # Make sure formatting of: LOAD_CONST False. We assume False is the 0th index
     # of co_consts.
     t = Token('LOAD_CONST', offset=1, attr=False, pattr=False, has_arg=True)
-    expect = '           1  LOAD_CONST            0  False'
+    expect = '           1  LOAD_CONST               False'
     assert t.format() == expect
 
 if __name__ == '__main__':

pytest/testdata/if-2.7.right

@@ -8,5 +8,5 @@
            9  STORE_NAME            2  'b'
           12  JUMP_FORWARD          0  'to 15'
         15_0  COME_FROM            12  '12'
-          15  LOAD_CONST            0  None
+          15  LOAD_CONST               None
           18  RETURN_VALUE

pytest/testdata/ifelse-2.7.right

@@ -4,12 +4,12 @@
    3       0  LOAD_NAME             0  'True'
            3  POP_JUMP_IF_FALSE    15  'to 15'
 
-   4       6  LOAD_CONST            0  1
+   4       6  LOAD_CONST               1
            9  STORE_NAME            1  'b'
           12  JUMP_FORWARD          6  'to 21'
 
-   6      15  LOAD_CONST            1  2
+   6      15  LOAD_CONST               2
           18  STORE_NAME            2  'd'
         21_0  COME_FROM            12  '12'
-          21  LOAD_CONST            2  None
+          21  LOAD_CONST               None
           24  RETURN_VALUE

test/simple_source/bug31/04_def_annotate.py

@@ -47,11 +47,50 @@ def div(a: dict(type=float, help='the dividend'),
     """Divide a by b"""
     return a / b
 
-# FIXME:
+class TestSignatureObject1():
-# class TestSignatureObject():
+    def test_signature_on_wkwonly(self):
-#     def test_signature_on_wkwonly(self):
+        def test(*, a:float, b:str, c:str = 'test', **kwargs: int) -> int:
-#         def test(*, a:float, b:str) -> int:
+            pass
-#             pass
+
+class TestSignatureObject2():
+    def test_signature_on_wkwonly(self):
+        def test(*, c='test', a:float, b:str="S", **kwargs: int) -> int:
+            pass
+
+class TestSignatureObject3():
+    def test_signature_on_wkwonly(self):
+        def test(*, c='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject4():
+    def test_signature_on_wkwonly(self):
+        def test(x=55, *args, c:str='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject5():
+    def test_signature_on_wkwonly(self):
+        def test(x=55, *args: int, c='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject5():
+    def test_signature_on_wkwonly(self):
+        def test(x:int=55, *args: (int, str), c='test', a:float, kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject7():
+    def test_signature_on_wkwonly(self):
+        def test(c='test', kwargs:str="S", **b: int) -> int:
+            pass
+
+class TestSignatureObject8():
+    def test_signature_on_wkwonly(self):
+        def test(**b: int) -> int:
+            pass
+
+class TestSignatureObject9():
+    def test_signature_on_wkwonly(self):
+        def test(a, **b: int) -> int:
+            pass
 
 class SupportsInt():
 

test/simple_source/bug36/05_ann_mopdule2.py

@@ -0,0 +1,37 @@
+# This is from Python 3.6's test directory.
+"""
+Some correct syntax for variable annotation here.
+More examples are in test_grammar and test_parser.
+"""
+
+from typing import no_type_check, ClassVar
+
+i: int = 1
+j: int
+x: float = i/10
+
+def f():
+    class C: ...
+    return C()
+
+f().new_attr: object = object()
+
+class C:
+    def __init__(self, x: int) -> None:
+        self.x = x
+
+c = C(5)
+c.new_attr: int = 10
+
+__annotations__ = {}
+
+
+@no_type_check
+class NTC:
+    def meth(self, param: complex) -> None:
+        ...
+
+class CV:
+   var: ClassVar['CV']
+
+CV.var = CV()

uncompyle6/parsers/parse3.py

@@ -114,10 +114,9 @@ class Python3Parser(PythonParser):
         continues ::= continue
 
 
-        kwarg      ::= LOAD_CONST expr
+        kwarg      ::= LOAD_STR expr
         kwargs     ::= kwarg+
 
-
         classdef ::= build_class store
 
         # FIXME: we need to add these because don't detect this properly
@@ -396,11 +395,12 @@ class Python3Parser(PythonParser):
     def p_generator_exp3(self, args):
         '''
         load_genexpr ::= LOAD_GENEXPR
-        load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_CONST
+        load_genexpr ::= BUILD_TUPLE_1 LOAD_GENEXPR LOAD_STR
         '''
 
     def p_expr3(self, args):
         """
+        expr           ::= LOAD_STR
         expr           ::= conditionalnot
         conditionalnot ::= expr jmp_true  expr jump_forward_else expr COME_FROM
 
@@ -443,7 +443,7 @@ class Python3Parser(PythonParser):
                 break
             pass
         assert i < len(tokens), "build_class needs to find MAKE_FUNCTION or MAKE_CLOSURE"
-        assert tokens[i+1].kind == 'LOAD_CONST', \
+        assert tokens[i+1].kind == 'LOAD_STR', \
           "build_class expecting CONST after MAKE_FUNCTION/MAKE_CLOSURE"
         call_fn_tok = None
         for i in range(i, len(tokens)):
@@ -517,13 +517,13 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
     def add_make_function_rule(self, rule, opname, attr, customize):
-        """Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and
+        """Python 3.3 added a an addtional LOAD_STR before MAKE_FUNCTION and
         this has an effect on many rules.
         """
         if self.version >= 3.3:
-            new_rule = rule % (('LOAD_CONST ') * 1)
+            new_rule = rule % (('LOAD_STR ') * 1)
         else:
-            new_rule = rule % (('LOAD_CONST ') * 0)
+            new_rule = rule % (('LOAD_STR ') * 0)
         self.add_unique_rule(new_rule, opname, attr, customize)
 
     def customize_grammar_rules(self, tokens, customize):
@@ -732,7 +732,7 @@ class Python3Parser(PythonParser):
 
                 if opname == 'CALL_FUNCTION' and token.attr == 1:
                     rule = """
-                     dict_comp    ::= LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_0 expr
+                     dict_comp    ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
                                       GET_ITER CALL_FUNCTION_1
                     classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
                     """
@@ -851,7 +851,7 @@ class Python3Parser(PythonParser):
                     # Note that 3.6+ doesn't do this, but we'll remove
                     # this rule in parse36.py
                     rule = """
-                        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
+                        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
                                       MAKE_CLOSURE_0 expr
                                       GET_ITER CALL_FUNCTION_1
                     """
@@ -904,10 +904,10 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= %s%sload_closure LOAD_CONST %s'
                             % (kwargs_str, 'expr ' * args_pos, opname))
                 elif self.version == 3.3:
-                    rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_CONST %s'
+                    rule = ('mkfunc ::= %s%sload_closure LOAD_CONST LOAD_STR %s'
                             % (kwargs_str, 'expr ' * args_pos, opname))
                 elif self.version >= 3.4:
-                    rule = ('mkfunc ::= %s%s load_closure LOAD_CONST LOAD_CONST %s'
+                    rule = ('mkfunc ::= %s%s load_closure LOAD_CONST LOAD_STR %s'
                             % ('expr ' * args_pos, kwargs_str, opname))
 
                 self.add_unique_rule(rule, opname, token.attr, customize)
@@ -935,17 +935,17 @@ class Python3Parser(PythonParser):
                             rule = ('mklambda ::= %s%s%s%s' %
                                         ('expr ' * stack_count,
                                          'load_closure ' * closure,
-                                         'BUILD_TUPLE_1 LOAD_LAMBDA LOAD_CONST ',
+                                         'BUILD_TUPLE_1 LOAD_LAMBDA LOAD_STR ',
                                         opname))
                         else:
                             rule = ('mklambda ::= %s%s%s' %
                                         ('load_closure ' * closure,
-                                         'LOAD_LAMBDA LOAD_CONST ',
+                                         'LOAD_LAMBDA LOAD_STR ',
                                         opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
 
                     else:
-                        rule = ('mklambda ::= %sLOAD_LAMBDA LOAD_CONST %s' %
+                        rule = ('mklambda ::= %sLOAD_LAMBDA LOAD_STR %s' %
                                 (('expr ' * stack_count), opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
 
@@ -953,7 +953,7 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= %s%s%s%s' %
                             ('expr ' * stack_count,
                              'load_closure ' * closure,
-                             'LOAD_CONST ' * 2,
+                             'LOAD_CONST LOAD_STR ',
                              opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
 
@@ -1035,17 +1035,17 @@ class Python3Parser(PythonParser):
                 elif self.version == 3.3:
                     # positional args after keyword args
                     rule = ('mkfunc ::= %s %s%s%s' %
-                            (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                            (kwargs, 'pos_arg ' * args_pos, 'LOAD_CONST LOAD_STR ',
                              opname))
                 elif self.version > 3.5:
                     # positional args before keyword args
                     rule = ('mkfunc ::= %s%s %s%s' %
-                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
+                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST LOAD_STR ',
                              opname))
                 elif self.version > 3.3:
                     # positional args before keyword args
                     rule = ('mkfunc ::= %s%s %s%s' %
-                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST '*2,
+                            ('pos_arg ' * args_pos, kwargs, 'LOAD_CONST LOAD_STR ',
                              opname))
                 else:
                     rule = ('mkfunc ::= %s%sexpr %s' %
@@ -1054,38 +1054,38 @@ class Python3Parser(PythonParser):
 
                 if re.search('^MAKE_FUNCTION.*_A', opname):
                     if self.version >= 3.6:
-                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_STR %s' %
                                 (('pos_arg ' * (args_pos)),
                                  ('call ' * (annotate_args-1)), opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
-                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST %s' %
+                        rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_STR %s' %
                                 (('pos_arg ' * (args_pos)),
                                  ('annotate_arg ' * (annotate_args-1)), opname))
                     if self.version >= 3.3:
                         # Normally we remove EXTENDED_ARG from the opcodes, but in the case of
                         # annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
                         # Yes this is a little hacky
-                        if self.version < 3.5:
+                        if self.version == 3.3:
-                            # 3.3 and 3.4 put kwargs before pos_arg
+                            # 3.3 puts kwargs before pos_arg
                             pos_kw_tuple = (('kwargs ' * args_kw), ('pos_arg ' * (args_pos)))
                         else:
-                            # 3.5 puts pos_arg before kwargs
+                            # 3.4 and 3.5puts pos_arg before kwargs
                             pos_kw_tuple = (('pos_arg ' * (args_pos), ('kwargs ' * args_kw)))
-                        rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                        rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST LOAD_STR EXTENDED_ARG %s' %
                                 ( pos_kw_tuple[0], pos_kw_tuple[1],
                                  ('call ' * (annotate_args-1)), opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
-                        rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
+                        rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST LOAD_STR EXTENDED_ARG %s' %
                                 ( pos_kw_tuple[0], pos_kw_tuple[1],
                                  ('annotate_arg ' * (annotate_args-1)), opname))
                     else:
                         # See above comment about use of EXTENDED_ARG
                         rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                                (('pos_arg ' * (args_pos)),  ('kwargs ' * args_kw),
+                                (('kwargs ' * args_kw), ('pos_arg ' * (args_pos)),
                                  ('annotate_arg ' * (annotate_args-1)), opname))
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = ('mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
-                                (('pos_arg ' * (args_pos)),  ('kwargs ' * args_kw),
+                                (('kwargs ' * args_kw), ('pos_arg ' * (args_pos)),
                                  ('call ' * (annotate_args-1)), opname))
                     self.addRule(rule, nop_func)
             elif opname == 'RETURN_VALUE_LAMBDA':
@@ -1151,6 +1151,7 @@ class Python3Parser(PythonParser):
         self.check_reduce['while1elsestmt'] = 'noAST'
         self.check_reduce['ifelsestmt'] = 'AST'
         self.check_reduce['annotate_tuple'] = 'noAST'
+        if not PYTHON3:
             self.check_reduce['kwarg'] = 'noAST'
         if self.version < 3.6:
             # 3.6+ can remove a JUMP_FORWARD which messes up our testing here
@@ -1168,9 +1169,6 @@ class Python3Parser(PythonParser):
             return not isinstance(tokens[first].attr, tuple)
         elif lhs == 'kwarg':
             arg = tokens[first].attr
-            if PYTHON3:
-                return not isinstance(arg, str)
-            else:
             return not (isinstance(arg, str) or isinstance(arg, unicode))
         elif lhs == 'while1elsestmt':
 

uncompyle6/parsers/parse36.py

@@ -29,8 +29,7 @@ class Python36Parser(Python35Parser):
 
 
     def p_36misc(self, args):
-        """
+        """sstmt ::= sstmt RETURN_LAST
-        sstmt ::= sstmt RETURN_LAST
 
         # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
         return_closure   ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
@@ -142,6 +141,7 @@ class Python36Parser(Python35Parser):
                                    COME_FROM_FINALLY
 
         compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
+
         """
 
     def customize_grammar_rules(self, tokens, customize):
@@ -201,14 +201,14 @@ class Python36Parser(Python35Parser):
                 if 'LOAD_DICTCOMP' in self.seen_ops:
                     # Is there something general going on here?
                     rule = """
-                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST
+                       dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
                                      MAKE_FUNCTION_8 expr
                                      GET_ITER CALL_FUNCTION_1
                        """
                     self.addRule(rule, nop_func)
                 elif 'LOAD_SETCOMP' in self.seen_ops:
                     rule = """
-                       set_comp ::= load_closure LOAD_SETCOMP LOAD_CONST
+                       set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
                                     MAKE_FUNCTION_8 expr
                                     GET_ITER CALL_FUNCTION_1
                        """
@@ -263,6 +263,23 @@ class Python36Parser(Python35Parser):
                 self.addRule(rule, nop_func)
                 rule = ('starred ::= %s %s' % ('expr ' * v, opname))
                 self.addRule(rule, nop_func)
+            elif opname == 'SETUP_ANNOTATIONS':
+                # 3.6 Variable Annotations PEP 526
+                # This seems to come before STORE_ANNOTATION, and doesn't
+                # correspond to direct Python source code.
+                rule = """
+                    stmt ::= SETUP_ANNOTATIONS
+                    stmt ::= ann_assign_init_value
+                    stmt ::= ann_assign_no_init
+
+                    ann_assign_init_value ::= expr store store_annotation
+                    ann_assign_no_init    ::= store_annotation
+                    store_annotation      ::= LOAD_NAME STORE_ANNOTATION
+                    store_annotation      ::= subscript STORE_ANNOTATION
+                 """
+                self.addRule(rule, nop_func)
+                # Check to combine assignment + annotation into one statement
+                self.check_reduce['assign'] = 'token'
             elif opname == 'SETUP_WITH':
                 rules_str = """
                 withstmt   ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
@@ -288,6 +305,7 @@ class Python36Parser(Python35Parser):
                 self.addRule(rules_str, nop_func)
                 pass
             pass
+        return
 
     def custom_classfunc_rule(self, opname, token, customize, next_token):
 
@@ -387,6 +405,15 @@ class Python36Parser(Python35Parser):
                                                 tokens, first, last)
         if invalid:
             return invalid
+        if rule[0] == 'assign':
+            # Try to combine assignment + annotation into one statement
+            if (len(tokens) >= last + 1 and
+                tokens[last] == 'LOAD_NAME' and
+                tokens[last+1] == 'STORE_ANNOTATION' and
+                tokens[last-1].pattr == tokens[last+1].pattr):
+                # Will handle as ann_assign_init_value
+                return True
+            pass
         if rule[0] == 'call_kw':
             # Make sure we don't derive call_kw
             nt = ast[0]

uncompyle6/scanners/scanner3.py

@@ -318,6 +318,8 @@ class Scanner3(Scanner):
                     # pattr = 'code_object @ 0x%x %s->%s' %\
                     # (id(const), const.co_filename, const.co_name)
                     pattr = '<code_object ' + const.co_name + '>'
+                elif isinstance(const, str):
+                    opname = 'LOAD_STR'
                 else:
                     if isinstance(inst.arg, int) and inst.arg < len(co.co_consts):
                         argval, _ = _get_const_info(inst.arg, co.co_consts)

uncompyle6/scanners/tok.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2018 by Rocky Bernstein
+#  Copyright (c) 2016-2019 by Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 #
@@ -58,7 +58,10 @@ class Token:   # Python 2.4 can't have empty ()
         """ '==' on kind and "pattr" attributes.
             It is okay if offsets and linestarts are different"""
         if isinstance(o, Token):
-            return (self.kind == o.kind) and (self.pattr == o.pattr)
+            return (
+                (self.kind == o.kind)
+                and ((self.pattr == o.pattr) or self.attr == o.attr)
+                )
         else:
             # ?? do we need this?
             return self.kind == o
@@ -85,13 +88,15 @@ class Token:   # Python 2.4 can't have empty ()
         else:
             prefix = ' ' * (6 + len(line_prefix))
         offset_opname = '%6s  %-17s' % (self.offset, self.kind)
+
         if not self.has_arg:
             return "%s%s" % (prefix, offset_opname)
-
         if isinstance(self.attr, int):
             argstr = "%6d " % self.attr
         else:
             argstr = ' '*7
+        name = self.kind
+
         if self.has_arg:
             pattr = self.pattr
             if self.opc:
@@ -104,13 +109,25 @@ class Token:   # Python 2.4 can't have empty ()
                         pattr = "to " + str(self.pattr)
                     pass
                 elif self.op in self.opc.CONST_OPS:
-                    # Compare with pysource n_LOAD_CONST
+                    if name == 'LOAD_STR':
-                    attr = self.attr
+                        pattr = self.attr
-                    if attr is None:
+                    elif name == 'LOAD_CODE':
-                        pattr = None
+                        return "%s%s%s %s" % (prefix, offset_opname,  argstr, pattr)
+                    else:
+                        return "%s%s        %r" % (prefix, offset_opname,  pattr)
+
                 elif self.op in self.opc.hascompare:
                     if isinstance(self.attr, int):
                         pattr = self.opc.cmp_op[self.attr]
+                    return "%s%s%s %s" % (prefix, offset_opname,  argstr, pattr)
+                elif self.op in self.opc.hasvargs:
+                    return "%s%s%s" % (prefix, offset_opname,  argstr)
+                elif self.op in self.opc.NAME_OPS:
+                    if self.opc.version >= 3.0:
+                        return "%s%s%s %s" % (prefix, offset_opname,  argstr, self.attr)
+                elif name == 'EXTENDED_ARG':
+                    return "%s%s%s 0x%x << %s = %s" % (prefix, offset_opname,  argstr, self.attr,
+                                                       self.opc.EXTENDED_ARG_SHIFT, pattr)
                 # And so on. See xdis/bytecode.py get_instructions_bytes
                 pass
         elif re.search(r'_\d+$', self.kind):

uncompyle6/semantics/consts.py

@@ -128,10 +128,10 @@ PASS = SyntaxTree('stmts',
                  [ SyntaxTree('stmt',
                        [ SyntaxTree('pass', [])])])])
 
-ASSIGN_DOC_STRING = lambda doc_string: \
+ASSIGN_DOC_STRING = lambda doc_string, doc_load: \
   SyntaxTree('stmt',
       [ SyntaxTree('assign',
-            [ SyntaxTree('expr', [ Token('LOAD_CONST', pattr=doc_string) ]),
+            [ SyntaxTree('expr', [ Token(doc_load, pattr=doc_string, attr=doc_string) ]),
               SyntaxTree('store', [ Token('STORE_NAME', pattr='__doc__')])
             ])])
 
@@ -221,6 +221,7 @@ TABLE_DIRECT = {
     'IMPORT_FROM':              ( '%{pattr}', ),
     'attribute':	        ( '%c.%[1]{pattr}',
                                   (0, 'expr')),
+    'LOAD_STR':	                ( '%{pattr}', ),
     'LOAD_FAST':	        ( '%{pattr}', ),
     'LOAD_NAME':	        ( '%{pattr}', ),
     'LOAD_CLASSNAME':	        ( '%{pattr}', ),
@@ -317,7 +318,7 @@ TABLE_DIRECT = {
     'mkfuncdeco0':  	    ( '%|def %c\n', 0),
     'classdefdeco':  	    ( '\n\n%c', 0),
     'classdefdeco1':  	    ( '%|@%c\n%c', 0, 1),
-    'kwarg':    	    ( '%[0]{pattr}=%c', 1),
+    'kwarg':    	    ( '%[0]{pattr}=%c', 1),  # Change when Python 2 does LOAD_STR
     'kwargs':    	    ( '%D', (0, maxint, ', ') ),
     'kwargs1':    	    ( '%D', (0, maxint, ', ') ),
 

uncompyle6/semantics/customize3.py

@@ -41,6 +41,7 @@ def customize_for_version3(self, version):
 
         'importmultiple' : ( '%|import %c%c\n', 2, 3 ),
         'import_cont'    : ( ', %c', 2 ),
+        'kwarg'          : ( '%[0]{attr}=%c', 1),
         'raise_stmt2'    : ( '%|raise %c from %c\n', 0, 1),
         'store_locals'   : ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
         'withstmt'       : ( '%|with %c:\n%+%c%-', 0, 3),
@@ -62,11 +63,11 @@ def customize_for_version3(self, version):
         subclass_info = None
         if node == 'classdefdeco2':
             if self.version >= 3.6:
-                class_name = node[1][1].pattr
+                class_name = node[1][1].attr
             elif self.version <= 3.3:
-                class_name = node[2][0].pattr
+                class_name = node[2][0].attr
             else:
-                class_name = node[1][2].pattr
+                class_name = node[1][2].attr
             build_class = node
         else:
             build_class = node[0]
@@ -87,7 +88,7 @@ def customize_for_version3(self, version):
                     code_node = build_class[1][0]
                 class_name = code_node.attr.co_name
             else:
-                class_name = node[1][0].pattr
+                class_name = node[1][0].attr
                 build_class = node[0]
 
         assert 'mkfunc' == build_class[1]

uncompyle6/semantics/customize36.py

@@ -60,6 +60,15 @@ def customize_for_version36(self, version):
         'call_ex' : (
             '%c(%p)',
             (0, 'expr'), (1, 100)),
+        'store_annotation': (
+            '%[1]{pattr}: %c',
+            0
+            ),
+        'ann_assign_init_value':  (
+            '%|%c = %p\n',
+             (-1, 'store_annotation'), (0, 'expr', 200)),
+        'ann_assign_no_init':  (
+            '%|%c\n', (0, 'store_annotation')),
 
     })
 
@@ -77,7 +86,7 @@ def customize_for_version36(self, version):
             self.call36_tuple(n)
             first = 1
             sep = ', *'
-        elif n == 'LOAD_CONST':
+        elif n == 'LOAD_STR':
             value = self.format_pos_args(n)
             self.f.write(value)
             first = 1
@@ -401,7 +410,7 @@ def customize_for_version36(self, version):
     self.n_except_suite_finalize = n_except_suite_finalize
 
     def n_formatted_value(node):
-        if node[0] == 'LOAD_CONST':
+        if node[0] in ('LOAD_STR', 'LOAD_CONST'):
             value = node[0].attr
             if isinstance(value, tuple):
                 self.write(node[0].attr)
@@ -415,7 +424,7 @@ def customize_for_version36(self, version):
     def n_formatted_value_attr(node):
         f_conversion(node)
         fmt_node = node.data[3]
-        if fmt_node == 'expr' and fmt_node[0] == 'LOAD_CONST':
+        if fmt_node == 'expr' and fmt_node[0] == 'LOAD_STR':
             node.string = escape_format(fmt_node[0].attr)
         else:
             node.string = fmt_node
@@ -424,7 +433,7 @@ def customize_for_version36(self, version):
 
     def f_conversion(node):
         fmt_node = node.data[1]
-        if fmt_node == 'expr' and fmt_node[0] == 'LOAD_CONST':
+        if fmt_node == 'expr' and fmt_node[0] == 'LOAD_STR':
             data = fmt_node[0].attr
         else:
             data = fmt_node.attr
@@ -482,11 +491,11 @@ def customize_for_version36(self, version):
             else:
                 # {{ and }} in Python source-code format strings mean
                 # { and } respectively. But only when *not* part of a
-                # formatted value. However in the LOAD_CONST
+                # formatted value. However in the LOAD_STR
                 # bytecode, the escaping of the braces has been
                 # removed. So we need to put back the braces escaping in
                 # reconstructing the source.
-                assert expr[0] == 'LOAD_CONST'
+                assert expr[0] == 'LOAD_STR'
                 value = value.replace("{", "{{").replace("}", "}}")
 
             # Remove leading quotes

uncompyle6/semantics/fragments.py

@@ -424,6 +424,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
             pass
         self.set_pos_info(node, start, len(self.f.getvalue()))
         self.prune()
+    n_LOAD_STR = n_LOAD_CONST
 
     def n_exec_stmt(self, node):
         """

uncompyle6/semantics/make_function.py

@@ -85,6 +85,12 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
         annotate_argc = 0
         pass
 
+    annotate_dict = {}
+
+    for name in annotate_args.keys():
+        n = self.traverse(annotate_args[name], indent='')
+        annotate_dict[name] = n
+
     if 3.0 <= self.version <= 3.2:
         lambda_index = -2
     elif 3.03 <= self.version:
@@ -103,7 +109,11 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
 
     # add defaults values to parameter names
     argc = code.co_argcount
+    kwonlyargcount = code.co_kwonlyargcount
+
     paramnames = list(code.co_varnames[:argc])
+    if kwonlyargcount > 0:
+        kwargs = list(code.co_varnames[argc:argc+kwonlyargcount])
 
     try:
         ast = self.build_ast(code._tokens,
@@ -137,14 +147,8 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
     for param in paramnames[:i]:
         self.write(suffix, param)
         suffix = ', '
-        if param in annotate_tuple[0].attr:
+        if param in annotate_dict:
-            # p = [x for x in annotate_tuple[0].attr].index(param)
+            self.write(': %s' % annotate_dict[param])
-            l = []
-            for x in annotate_tuple[0].attr:
-                l.append(x)
-            p = l.index(param)
-            self.write(': ')
-            self.preorder(node[p])
             if (line_number != self.line_number):
                 suffix = ",\n" + indent
                 line_number = self.line_number
@@ -184,17 +188,16 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
 
 
     if code_has_star_arg(code):
-        star_arg = code.co_varnames[argc + kw_pairs]
+        star_arg = code.co_varnames[argc + kwonlyargcount]
+        if annotate_dict and star_arg in annotate_dict:
+            self.write(suffix, '*%s: %s' % (star_arg, annotate_dict[star_arg]))
+        else:
             self.write(suffix, '*%s' % star_arg)
-        if star_arg in annotate_tuple[0].attr:
-            p = annotate_tuple[0].attr.index(star_arg) + pos_args + kw_args
-            self.write(': ')
-            self.preorder(node[p])
         argc += 1
 
     # self.println(indent, '#flags:\t', int(code.co_flags))
     ends_in_comma = False
-    if kw_args + annotate_argc > 0:
+    if kwonlyargcount > 0:
         if no_paramnames:
             if not code_has_star_arg(code):
                 if argc > 0:
@@ -205,46 +208,51 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
             else:
                 self.write(", ")
             ends_in_comma = True
-
-        kwargs = node[1]
-        last = len(kwargs)-1
-        i = 0
-        for n in node[1]:
-            if n == 'kwarg':
-                if argc > 0 and not ends_in_comma:
-                    self.write(', ')
-                if (line_number != self.line_number):
-                    self.write("\n" + indent)
-                    line_number = self.line_number
-                kn = n[0].pattr
-                if kn in annotate_tuple[0].attr:
-                    p = annotate_tuple[0].attr.index(star_arg) + pos_args + kw_args
-                    self.write('%s: ' % kn)
-                    self.preorder(node[p])
-                    self.write('=')
         else:
-                    self.write('%s=' % kn)
+            if argc > 0:
-                self.preorder(n[1])
-                if i < last:
                 self.write(', ')
                 ends_in_comma = True
+
+        kw_args = [None] * kwonlyargcount
+
+        for n in node:
+            if n == 'kwargs':
+                n = n[0]
+            if n == 'kwarg':
+                name = eval(n[0].pattr)
+                idx = kwargs.index(name)
+                default = self.traverse(n[1], indent='')
+                if annotate_dict and name in annotate_dict:
+                    kw_args[idx] = '%s: %s=%s' % (name, annotate_dict[name], default)
                 else:
-                    ends_in_comma = False
+                    kw_args[idx] = '%s=%s' % (name, default)
-                i += 1
-                pass
                 pass
             pass
 
+        # handling other args
+        ann_other_kw = [c == None for c in kw_args]
+        for i, flag in enumerate(ann_other_kw):
+            if flag:
+                n = kwargs[i]
+                if n in annotate_dict:
+                    kw_args[i] = "%s: %s" %(n, annotate_dict[n])
+                else:
+                    kw_args[i] = "%s" % n
+
+        self.write(', '.join(kw_args), ', ')
+
+    else:
+        if argc == 0:
+            ends_in_comma = True
 
     if code_has_star_star_arg(code):
-            if argc > 0 and not ends_in_comma:
+        if not ends_in_comma:
             self.write(', ')
-            star_star_arg = code.co_varnames[argc + kw_pairs]
+        star_star_arg = code.co_varnames[argc + kwonlyargcount]
+        if annotate_dict and star_star_arg in annotate_dict:
+            self.write('**%s: %s' % (star_star_arg, annotate_dict[star_star_arg]))
+        else:
             self.write('**%s' % star_star_arg)
-            if star_star_arg in annotate_tuple[0].attr:
-                p = annotate_tuple[0].attr.index(star_star_arg) + pos_args + kw_args
-                self.write(': ')
-                self.preorder(node[p])
 
     if is_lambda:
         self.write(": ")
@@ -669,7 +677,11 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     # add defaults values to parameter names
     argc = code.co_argcount
+    kwonlyargcount = code.co_kwonlyargcount
+
     paramnames = list(scanner_code.co_varnames[:argc])
+    if kwonlyargcount > 0:
+        kwargs = list(scanner_code.co_varnames[argc:argc+kwonlyargcount])
 
     # defaults are for last n parameters, thus reverse
     paramnames.reverse();
@@ -692,6 +704,9 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     else:
         kw_pairs = 0
 
+    i = len(paramnames) - len(defparams)
+    no_paramnames = len(paramnames[:i]) == 0
+
     # build parameters
     params = []
     if defparams:
@@ -715,9 +730,9 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
 
     if code_has_star_arg(code):
         if self.version > 3.0:
-            star_arg = code.co_varnames[argc + kw_pairs]
+            star_arg = code.co_varnames[argc + kwonlyargcount]
-            if star_arg in annotate_dict:
+            if annotate_dict and star_arg in annotate_dict:
-                params.append('*%s: %s' %(star_arg, annotate_dict[star_arg]))
+                params.append('*%s: %s' % (star_arg, annotate_dict[star_arg]))
             else:
                 params.append('*%s' % star_arg)
         else:
@@ -744,11 +759,16 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             ast[-1] = ast_expr
             pass
     else:
+        # FIXME: add annotations here
         self.write("(", ", ".join(params))
     # self.println(indent, '#flags:\t', int(code.co_flags))
 
+    # FIXME: Could we remove ends_in_comma and its tests if we just
+    # created a parameter list and at the very end did a join on that?
+    # Unless careful, We might lose line breaks though.
     ends_in_comma = False
-    if kw_args > 0:
+    if kwonlyargcount > 0:
+        if no_paramnames:
             if not (4 & code.co_flags):
                 if argc > 0:
                     self.write(", *, ")
@@ -758,6 +778,10 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             else:
                 self.write(", ")
                 ends_in_comma = True
+        else:
+            if argc > 0:
+                self.write(', ')
+                ends_in_comma = True
 
         # FIXME: this is not correct for 3.5. or 3.6 (which works different)
         # and 3.7?
@@ -767,7 +791,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             i = 0
             for n in node[0]:
                 if n == 'kwarg':
-                    self.write('%s=' % n[0].pattr)
+                    self.write('%s=' % n[0].attr)
                     self.preorder(n[1])
                     if i < last:
                         self.write(', ')
@@ -796,7 +820,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
             # argcount = co.co_argcount
             # kwonlyargcount = co.co_kwonlyargcount
 
-            free_tup = annotate_dict = kw_dict = default_tup = None
+            free_tup = ann_dict = kw_dict = default_tup = None
             fn_bits = node[-1].attr
             index = -4  # Skip over:
                         #  MAKE_FUNCTION,
@@ -806,7 +830,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 free_tup = node[index]
                 index -= 1
             if fn_bits[-2]:
-                annotate_dict = node[index]
+                ann_dict = node[index]
                 index -= 1
             if fn_bits[-3]:
                 kw_dict = node[index]
@@ -818,6 +842,8 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 kw_dict = kw_dict[0]
 
             # FIXME: handle free_tup, annotate_dict, and default_tup
+            kw_args = [None] * kwonlyargcount
+
             if kw_dict:
                 assert kw_dict == 'dict'
                 defaults = [self.traverse(n, indent='') for n in kw_dict[:-2]]
@@ -826,20 +852,40 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
                 sep = ''
                 # FIXME: possibly handle line breaks
                 for i, n in enumerate(names):
-                    self.write(sep)
+                    idx = kwargs.index(n)
-                    self.write("%s=%s" % (n, defaults[i]))
+                    if annotate_dict and n in annotate_dict:
-                    sep = ', '
+                        t = "%s: %s=%s" % (n, annotate_dict[n], defaults[i])
-                    ends_in_comma = False
+                    else:
-                    pass
+                        t = "%s=%s" % (n, defaults[i])
+                    kw_args[idx] = t
                     pass
                 pass
 
+            # handle others
+            if ann_dict:
+                ann_other_kw = [c == None for c in kw_args]
+
+                for i, flag in enumerate(ann_other_kw):
+                    if flag:
+                        n = kwargs[i]
+                        if n in annotate_dict:
+                            kw_args[i] = "%s: %s" %(n, annotate_dict[n])
+                        else:
+                            kw_args[i] = "%s" % n
+            self.write(', '.join(kw_args))
+            ends_in_comma = False
+
+        pass
+    else:
+        if argc == 0:
+            ends_in_comma = True
+
     if code_has_star_star_arg(code):
-        if argc > 0 and not ends_in_comma:
+        if not ends_in_comma:
             self.write(', ')
-        star_star_arg = code.co_varnames[argc + kw_pairs]
+        star_star_arg = code.co_varnames[argc + kwonlyargcount]
-        if annotate_dict and star_star_arg and star_star_arg in annotate_dict:
+        if annotate_dict and star_star_arg in annotate_dict:
-            self.write('**%s: %s' %(star_star_arg, annotate_dict[star_star_arg]))
+            self.write('**%s: %s' % (star_star_arg, annotate_dict[star_star_arg]))
         else:
             self.write('**%s' % star_star_arg)
 

uncompyle6/semantics/pysource.py

@@ -1436,7 +1436,7 @@ class SourceWalker(GenericASTTraversal, object):
         n = len(node) - 1
         if node.kind != 'expr':
             if node == 'kwarg':
-                self.template_engine(('(%[0]{pattr}=%c)', 1), node)
+                self.template_engine(('(%[0]{attr}=%c)', 1), node)
                 return
 
             kwargs = None
@@ -2107,6 +2107,7 @@ class SourceWalker(GenericASTTraversal, object):
         except:
             pass
 
+
         have_qualname = False
         if self.version < 3.0:
             # Should we ditch this in favor of the "else" case?
@@ -2122,7 +2123,7 @@ class SourceWalker(GenericASTTraversal, object):
             # which are not simple classes like the < 3 case.
             try:
                 if (first_stmt[0] == 'assign' and
-                    first_stmt[0][0][0] == 'LOAD_CONST' and
+                    first_stmt[0][0][0] == 'LOAD_STR' and
                     first_stmt[0][1] == 'store' and
                     first_stmt[0][1][0] == Token('STORE_NAME', pattr='__qualname__')):
                     have_qualname = True
@@ -2333,13 +2334,28 @@ def code_deparse(co, out=sys.stdout, version=None, debug_opts=DEFAULT_DEBUG_OPTS
 
     assert not nonlocals
 
+    if version >= 3.0:
+        load_op = 'LOAD_STR'
+    else:
+        load_op = 'LOAD_CONST'
+
     # convert leading '__doc__ = "..." into doc string
     try:
-        if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]):
+        stmts = deparsed.ast
+        first_stmt = stmts[0][0]
+        if version >= 3.6:
+            if first_stmt[0] == 'SETUP_ANNOTATIONS':
+                del stmts[0]
+                assert stmts[0] == 'sstmt'
+                # Nuke sstmt
+                first_stmt = stmts[0][0]
+                pass
+            pass
+        if first_stmt == ASSIGN_DOC_STRING(co.co_consts[0], load_op):
             print_docstring(deparsed, '', co.co_consts[0])
-            del deparsed.ast[0]
+            del stmts[0]
-        if deparsed.ast[-1] == RETURN_NONE:
+        if stmts[-1] == RETURN_NONE:
-            deparsed.ast.pop() # remove last node
+            stmts.pop() # remove last node
             # todo: if empty, add 'pass'
     except:
         pass

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.3.3'  # noqa
+VERSION='3.3.4'  # noqa