Merge branch 'master' of github.com:rocky/python-uncompyle6

uncompyle6/bin/uncompile.py

@@ -64,8 +64,8 @@ def usage():
 
 
 def main_bin():
-    if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 2), (3, 3), (3, 4), (3, 5))):
-        print('Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4 or 3.5' % program,
+    if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6))):
+        print('Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, or 3.6' % program,
               file=sys.stderr)
         sys.exit(-1)
 

uncompyle6/parsers/parse3.py

@@ -365,7 +365,7 @@ class Python3Parser(PythonParser):
         call_function ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
 
         classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n
-       """
+        """
         # Low byte indicates number of positional paramters,
         # high byte number of positional parameters
         args_pos = token.attr & 0xff
@@ -444,7 +444,8 @@ class Python3Parser(PythonParser):
         For PYPY:
             load_attr ::= expr LOOKUP_METHOD
             call_function ::= expr CALL_METHOD
-       """
+        """
+        saw_format_value = False
         for i, token in enumerate(tokens):
             opname = token.type
             opname_base = opname[:opname.rfind('_')]
@@ -457,8 +458,19 @@ class Python3Parser(PythonParser):
                     assign2_pypy ::= expr expr designator designator
                 """, nop_func)
                 continue
+            elif opname == 'FORMAT_VALUE':
+                # Python 3.6+
+                self.addRule("""
+                formatted_value ::= LOAD_FAST FORMAT_VALUE
+                formatted_value ::= LOAD_NAME FORMAT_VALUE
+                str ::= LOAD_CONST
+                formatted_value_or_str ::= formatted_value
+                formatted_value_or_str ::= str
+                """, nop_func)
+                saw_format_value = True
+
             elif opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR',
-                          'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
+                            'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'):
                 self.custom_classfunc_rule(opname, token, customize)
             elif opname == 'LOAD_DICTCOMP':
                 rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr "
@@ -479,6 +491,15 @@ class Python3Parser(PythonParser):
                 if opname_base == 'BUILD_TUPLE':
                     rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
                     self.add_unique_rule(rule, opname, token.attr, customize)
+                if opname_base == 'BUILD_LIST' and saw_format_value:
+                    format_or_str_n = "formatted_value_or_str_%s" % v
+                    self.addRule("""
+                    expr ::= joined_str
+                    joined_str ::=  LOAD_CONST LOAD_ATTR %s CALL_FUNCTION_1
+                    %s ::= %s%s
+                    """ % (format_or_str_n, format_or_str_n, ("formatted_value_or_str " *v), opname),
+                                 nop_func)
+
             elif opname == 'LOOKUP_METHOD':
                 # A PyPy speciality - DRY with parse2
                 self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",

uncompyle6/parsers/parse36.py

@@ -0,0 +1,52 @@
+#  Copyright (c) 2016 Rocky Bernstein
+"""
+spark grammar differences over Python 3.5 for Python 3.6.
+"""
+from __future__ import print_function
+
+from uncompyle6.parser import PythonParserSingle
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parsers.parse35 import Python35Parser
+
+class Python36Parser(Python35Parser):
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python36Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+    def p_36misc(self, args):
+        """
+        formatted_value ::= LOAD_FAST FORMAT_VALUE
+        str ::= LOAD_CONST
+        joined_str ::=  LOAD_CONST LOAD_ATTR format_value_or_strs
+                        BUILD_LIST CALL_FUNCTION
+        format_value_or_strs ::= format_value_or_strs format_value_or_str
+        format_value_or_strs ::= format_value_or_str
+        format_value_or_str ::= format_value
+        format_value_or_str ::= str
+        """
+
+class Python36ParserSingle(Python36Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python36Parser()
+    p.checkGrammar()
+    from uncompyle6 import PYTHON_VERSION, IS_PYPY
+    if PYTHON_VERSION == 3.6:
+        lhs, rhs, tokens, right_recursive = p.checkSets()
+        from uncompyle6.scanner import get_scanner
+        s = get_scanner(PYTHON_VERSION, IS_PYPY)
+        opcode_set = set(s.opc.opname).union(set(
+            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
+               LAMBDA_MARKER RETURN_LAST
+            """.split()))
+        remain_tokens = set(tokens) - opcode_set
+        import re
+        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        print(remain_tokens)
+        # print(sorted(p.rule2name.items()))

uncompyle6/scanner.py

@@ -22,7 +22,7 @@ from uncompyle6 import PYTHON3, IS_PYPY
 from uncompyle6.scanners.tok import Token
 
 # The byte code versions we support
-PYTHON_VERSIONS = (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5)
+PYTHON_VERSIONS = (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)
 
 # FIXME: DRY
 if PYTHON3:

uncompyle6/scanners/scanner36.py

@@ -0,0 +1,35 @@
+#  Copyright (c) 2016 by Rocky Bernstein
+"""
+Python 3.5 bytecode scanner/deparser
+
+This sets up opcodes Python's 3.5 and calls a generalized
+scanner routine for Python 3.
+"""
+
+from __future__ import print_function
+
+from uncompyle6.scanners.scanner3 import Scanner3
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_36 as opc
+JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs)
+
+class Scanner36(Scanner3):
+
+    def __init__(self, show_asm=None):
+        Scanner3.__init__(self, 3.6, show_asm)
+        return
+    pass
+
+if __name__ == "__main__":
+    from uncompyle6 import PYTHON_VERSION
+    if PYTHON_VERSION == 3.6:
+        import inspect
+        co = inspect.currentframe().f_code
+        tokens, customize = Scanner36().disassemble(co)
+        for t in tokens:
+            print(t.format())
+        pass
+    else:
+        print("Need to be Python 3.6 to demo; I am %s." %
+              PYTHON_VERSION)

uncompyle6/scanners/tok.py

@@ -2,7 +2,7 @@
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 
-import sys
+import re, sys
 from uncompyle6 import PYTHON3
 
 if PYTHON3:
@@ -71,8 +71,10 @@ class Token:
                 elif self.op in self.opc.hascompare:
                     if isinstance(self.attr, int):
                         pattr = self.opc.cmp_op[self.attr]
-                    # And so on. See xdis/bytecode.py get_instructions_bytes
+                # And so on. See xdis/bytecode.py get_instructions_bytes
                 pass
+        elif re.search('_\d+$', self.type):
+            return "%s%s%s" % (prefix, offset_opname,  argstr)
         else:
             pattr = ''
         return "%s%s%s %r" % (prefix, offset_opname,  argstr, pattr)

uncompyle6/semantics/aligner.py

@@ -0,0 +1,147 @@
+import sys
+from uncompyle6.semantics.pysource import (
+    SourceWalker, SourceWalkerError, find_globals, ASSIGN_DOC_STRING, RETURN_NONE)
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+class AligningWalker(SourceWalker, object):
+    def __init__(self, version, scanner, out, showast=False,
+                 debug_parser=PARSER_DEFAULT_DEBUG,
+                 compile_mode='exec', is_pypy=False):
+        SourceWalker.__init__(self, version, out, scanner, showast, debug_parser,
+                              compile_mode, is_pypy)
+        self.desired_line_number = 0
+        self.current_line_number = 0
+
+    def println(self, *data):
+        if data and not(len(data) == 1 and data[0] ==''):
+            self.write(*data)
+
+        self.pending_newlines = max(self.pending_newlines, 1)
+
+    def write(self, *data):
+        from trepan.api import debug; debug()
+        if (len(data) == 1) and data[0] == self.indent:
+            diff = max(self.pending_newlines,
+                       self.desired_line_number - self.current_line_number)
+            self.f.write('\n'*diff)
+            self.current_line_number += diff
+            self.pending_newlines = 0
+        if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
+            return
+
+        out = ''.join((str(j) for j in data))
+        n = 0
+        for i in out:
+            if i == '\n':
+                n += 1
+                if n == len(out):
+                    self.pending_newlines = max(self.pending_newlines, n)
+                    return
+            elif n:
+                self.pending_newlines = max(self.pending_newlines, n)
+                out = out[n:]
+                break
+            else:
+                break
+
+        if self.pending_newlines > 0:
+            diff = max(self.pending_newlines,
+                       self.desired_line_number - self.current_line_number)
+            self.f.write('\n'*diff)
+            self.current_line_number += diff
+            self.pending_newlines = 0
+
+        for i in out[::-1]:
+            if i == '\n':
+                self.pending_newlines += 1
+            else:
+                break
+
+        if self.pending_newlines:
+            out = out[:-self.pending_newlines]
+        self.f.write(out)
+
+    def default(self, node):
+        mapping = self._get_mapping(node)
+        if hasattr(node, 'linestart'):
+            if node.linestart:
+                self.desired_line_number = node.linestart
+        table = mapping[0]
+        key = node
+
+        for i in mapping[1:]:
+            key = key[i]
+            pass
+
+        if key.type in table:
+            self.engine(table[key.type], node)
+            self.prune()
+
+from xdis.code import iscode
+from uncompyle6.scanner import get_scanner
+from uncompyle6.show import (
+    maybe_show_asm,
+)
+
+def align_deparse_code(version, co, out=sys.stderr, showasm=False, showast=False,
+                 showgrammar=False, code_objects={}, compile_mode='exec', is_pypy=False):
+    """
+    disassembles and deparses a given code block 'co'
+    """
+
+    assert iscode(co)
+    # store final output stream for case of error
+    scanner = get_scanner(version, is_pypy=is_pypy)
+
+    tokens, customize = scanner.disassemble(co, code_objects=code_objects)
+    maybe_show_asm(showasm, tokens)
+
+    debug_parser = dict(PARSER_DEFAULT_DEBUG)
+    if showgrammar:
+        debug_parser['reduce'] = showgrammar
+        debug_parser['errorstack'] = True
+
+    #  Build AST from disassembly.
+    deparsed = AligningWalker(version, scanner, out, showast=showast,
+                            debug_parser=debug_parser, compile_mode=compile_mode,
+                            is_pypy = is_pypy)
+
+    isTopLevel = co.co_name == '<module>'
+    deparsed.ast = deparsed.build_ast(tokens, customize, isTopLevel=isTopLevel)
+
+    assert deparsed.ast == 'stmts', 'Should have parsed grammar start'
+
+    del tokens # save memory
+
+    deparsed.mod_globs = find_globals(deparsed.ast, set())
+
+    # convert leading '__doc__ = "..." into doc string
+    try:
+        if deparsed.ast[0][0] == ASSIGN_DOC_STRING(co.co_consts[0]):
+            deparsed.print_docstring('', co.co_consts[0])
+            del deparsed.ast[0]
+        if deparsed.ast[-1] == RETURN_NONE:
+            deparsed.ast.pop() # remove last node
+            # todo: if empty, add 'pass'
+    except:
+        pass
+
+    # What we've been waiting for: Generate source from AST!
+    deparsed.gen_source(deparsed.ast, co.co_name, customize)
+
+    for g in deparsed.mod_globs:
+        deparsed.write('# global %s ## Warning: Unused global' % g)
+
+    if deparsed.ERROR:
+        raise SourceWalkerError("Deparsing stopped due to parse error")
+    return deparsed
+
+if __name__ == '__main__':
+    def deparse_test(co):
+        "This is a docstring"
+        sys_version = sys.version_info.major + (sys.version_info.minor / 10.0)
+        # deparsed = deparse_code(sys_version, co, showasm=True, showast=True)
+        deparsed = align_deparse_code(sys_version, co, showasm=False, showast=False,
+                                      showgrammar=False)
+        print(deparsed.text)
+        return
+    deparse_test(deparse_test.__code__)

uncompyle6/semantics/pysource.py

@@ -586,6 +586,14 @@ class SourceWalker(GenericASTTraversal, object):
             TABLE_DIRECT.update({
                 'LOAD_CLASSDEREF':	( '%{pattr}', ),
                 })
+            if version >= 3.6:
+                ########################
+                # Python 3.6+ Additions
+                #######################
+                TABLE_DIRECT.update({
+                    'formatted_value': ( '{%c}', 0),
+                    'joined_str':  ( "f'%c'", 2),
+                })
         return
 
     f = property(lambda s: s.params['f'],
@@ -836,6 +844,10 @@ class SourceWalker(GenericASTTraversal, object):
         self.prec += 1
         self.prune()
 
+    def n_str(self, node):
+        self.write(node[0].pattr)
+        self.prune()
+
     def n_LOAD_CONST(self, node):
         data = node.pattr; datatype = type(data)
         if isinstance(datatype, int) and data == minint:

uncompyle6/verify.py

@@ -215,6 +215,9 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, name=''):
             elif version == 3.5:
                 import uncompyle6.scanners.scanner35 as scan
                 scanner = scan.Scanner35()
+            elif version == 3.6:
+                import uncompyle6.scanners.scanner36 as scan
+                scanner = scan.Scanner36()
 
             global JUMP_OPs
             JUMP_OPs = list(scan.JUMP_OPs) + ['JUMP_BACK']