Merge branch 'master' into python-2.4

NEWS

@@ -1,3 +1,10 @@
+uncompyle6 3.2.0 2018-05-19 Rocket Scientist
+
+- Add rudimentary 1.4 support (still a bit buggy)
+- add --tree+ option to show formatting rule, when it is constant
+- Python 2.7.15candidate1 support (via xdis)
+- bug fixes, especially for 3.7 (but 2.7 and 3.6 and others as well)
+
 uncompyle6 3.1.3 2018-04-16
 
 - Add some Python 3.7 rules, such as for handling LOAD_METHOD (not complete)

__pkginfo__.py

@@ -56,7 +56,7 @@ entry_points       = {
     ]}
 ftp_url            = None
 install_requires   = ['spark-parser >= 1.8.5, < 1.9.0',
-                      'xdis >= 3.7.0, < 3.8.0']
+                      'xdis >= 3.8.2, < 3.9.0']
 license            = 'GPL3'
 mailing_list       = 'python-debugger@googlegroups.com'
 modname            = 'uncompyle6'

circle.yml

@@ -6,6 +6,7 @@ machine:
 
 dependencies:
     override:
+        - pip install --upgrade setuptools
         - pip install -e .
         - pip install pytest==3.2.5 hypothesis
 test:

test/Makefile

@@ -1,5 +1,6 @@
 PHONY=check clean dist distclean test test-unit test-functional rmChangeLog clean_pyc nosetests \
-      check-bytecode-1.5 check-bytecode-1 check-bytecode-2 check-bytecode-3 \
+      check-bytecode-1  check-bytecode-1.4 check-bytecode-1.5 \
+      check-bytecode-2 check-bytecode-3 \
       check-bytecode-2.2 check-byteocde-2.3 check-bytecode-2.4 \
       check-short check-2.6 check-2.7 check-3.0 check-3.1 check-3.2 check-3.3 \
       check-3.4 check-3.5 check-3.6 check-3.7 check-5.6 5.6 5.8 \
@@ -76,7 +77,7 @@ check-disasm:
 	$(PYTHON) dis-compare.py
 
 #: Check deparsing bytecode 1.x only
-check-bytecode-1: check-bytecode-1.5
+check-bytecode-1: check-bytecode-1.4 check-bytecode-1.5
 
 #: Check deparsing bytecode 2.x only
 check-bytecode-2:
@@ -93,11 +94,17 @@ check-bytecode-3:
 #: Check deparsing bytecode that works running Python 2 and Python 3
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
+	--bytecode-1.4 --bytecode-1.5 \
+        --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
         --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode 1.4 only
+check-bytecode-1.4:
+	$(PYTHON) test_pythonlib.py --bytecode-1.4
+
 #: Check deparsing bytecode 1.5 only
 check-bytecode-1.5:
 	$(PYTHON) test_pythonlib.py --bytecode-1.5

test/simple_source/bug26/03_complex_and.py

@@ -0,0 +1,17 @@
+# From 2.6 test_datetime.py
+# Bug is in parsing (x is 0 or x is 1) and (y is 5 or y is 2)
+# correctly.
+
+# This code is RUNNABLE!
+result = []
+for y in (1, 2, 10):
+    x = cmp(1, y)
+    if (x is 0 or x is 1) and (y is 5 or y is 2):
+        expected = 10
+    elif y is 2:
+        expected = 2
+    else:
+        expected = 3
+    result.append(expected)
+
+assert result == [10, 2, 3]

test/simple_source/bug27+/05_try_else.py

@@ -0,0 +1,11 @@
+# From Python 2.7 test_ziplib.py
+# Bug is distinguishing try from try/else.
+def testAFakeZlib(self):
+    try:
+        self.doTest()
+    except ImportError:
+        if self.compression != 3:
+            self.fail("expected test to not raise ImportError")
+    else:
+        if self.compression != 4:
+            self.fail("expected test to raise ImportError")

test/simple_source/comprehension/05_dict_comp.py

@@ -1,4 +1,7 @@
+# Bug was in dictionary comprehension involving "if not"
 # Issue #162
+#
+# This code is RUNNABLE!
 def x(s):
     return {k: v
             for (k, v) in s

test/stdlib/runtests.sh

@@ -75,12 +75,24 @@ case $PYVERSION in
 	    # .pyenv/versions/2.6.9/lib/python2.6/sre_parse.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/tabnanny.pyc
 	    # .pyenv/versions/2.6.9/lib/python2.6/tarfile.pyc
+
+	    # Not getting set by bach below?
+	    [test_pprint.py]=1
+
 	)
 	if (( batch )) ; then
 	    # Fails in crontab environment?
 	    # Figure out what's up here
 	    SKIP_TESTS[test_aifc.py]=1
 	    SKIP_TESTS[test_array.py]=1
+
+	    # SyntaxError: Non-ASCII character '\xdd' in file test_base64.py on line 153, but no encoding declared; see http://www.python.org/peps/pep-0263.html for details
+	    SKIP_TESTS[test_base64.py]=1
+
+	    # output indicates expected == output, but this fails anyway.
+	    # Maybe the underlying encoding is subtlely different so it
+	    # looks the same?
+	    SKIP_TESTS[test_pprint.py]=1
 	fi
 	;;
     2.7)
@@ -111,6 +123,7 @@ case $PYVERSION in
 	    [test_unicode.py]=1  # Too long to run 11 seconds
 	    [test_xpickle.py]=1 # Runs ok but takes 72 seconds
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
+	    [test_zipimport.py]=1  # We can't distinguish try from try/else yet
         )
 	if (( batch )) ; then
 	    # Fails in crontab environment?
@@ -118,6 +131,9 @@ case $PYVERSION in
 	    SKIP_TESTS[test_array.py]=1
 	    SKIP_TESTS[test_ast.py]=1
 	    SKIP_TESTS[test_audioop.py]=1
+
+	    # SyntaxError: Non-ASCII character '\xdd' in file test_base64.py on line 153, but no encoding declared; see http://www.python.org/peps/pep-0263.html for details
+	    SKIP_TESTS[test_base64.py]=1
 	fi
 	;;
     3.5)

test/test_pythonlib.py

@@ -76,7 +76,7 @@ for  vers in (2.7, 3.4, 3.5, 3.6):
     test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers)
     pass
 
-for  vers in (1.5,
+for  vers in (1.4, 1.5,
               2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
               3.0, 3.1, 3.2, 3.3,
               3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'):

uncompyle6/bin/uncompile.py

@@ -45,6 +45,7 @@ Debugging Options:
   --asm     -a  include byte-code         (disables --verify)
   --grammar -g  show matching grammar
   --tree    -t  include syntax tree       (disables --verify)
+  --tree++      add template rules to --tree when possible
 
 Extensions of generated files:
   '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
@@ -60,7 +61,7 @@ from uncompyle6.version import VERSION
 
 def usage():
     print("""usage:
-   %s [--verify | --weak-verify ] [--asm] [--tree] [--grammar] [-o <path>] FILE|DIR...
+    %s [--verify | --weak-verify ] [--asm] [--tree[+]] [--grammar] [-o <path>] FILE|DIR...
    %s [--help | -h | --version | -V]
 """  % (program, program))
     sys.exit(1)
@@ -84,8 +85,10 @@ def main_bin():
 
     try:
         opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
-                                    'help asm grammar linemaps recurse timestamp tree '
+                                    'help asm grammar linemaps recurse '
-                                    'fragments verify verify-run version weak-verify '
+                                    'timestamp tree tree+ '
+                                    'fragments verify verify-run version '
+                                    'weak-verify '
                                     'showgrammar'.split(' '))
     except getopt.GetoptError(e):
         sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
@@ -115,6 +118,9 @@ def main_bin():
         elif opt in ('--tree', '-t'):
             options['showast'] = True
             options['do_verify'] = None
+        elif opt in ('--tree+',):
+            options['showast'] = 'Full'
+            options['do_verify'] = None
         elif opt in ('--grammar', '-g'):
             options['showgrammar'] = True
         elif opt == '-o':

uncompyle6/parsers/parse14.py

@@ -0,0 +1,31 @@
+#  Copyright (c) 2018 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parsers.parse15 import Python15Parser
+
+class Python14Parser(Python15Parser):
+
+    def p_misc14(self, args):
+        """
+        # Nothing here yet, but will need to add UNARY_CALL, BINARY_CALL,
+        # RAISE_EXCEPTION, BUILD_FUNCTION, UNPACK_ARG, UNPACK_VARARG, LOAD_LOCAL,
+        # SET_FUNC_ARGS, and RESERVE_FAST
+        """
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python14Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+
+class Python14ParserSingle(Python14Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python14Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4

uncompyle6/parsers/parse15.py

@@ -23,7 +23,7 @@ class Python15Parser(Python21Parser):
         importlist  ::= IMPORT_FROM
         """
 
-class Python15ParserSingle(Python21Parser, PythonParserSingle):
+class Python15ParserSingle(Python15Parser, PythonParserSingle):
     pass
 
 if __name__ == '__main__':

uncompyle6/scanner.py

@@ -39,7 +39,7 @@ else:
 
 # The byte code versions we support.
 # Note: these all have to be floats
-PYTHON_VERSIONS = frozenset((1.5,
+PYTHON_VERSIONS = frozenset((1.4, 1.5,
                              2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
                              3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7))
 
@@ -121,7 +121,10 @@ class Scanner(object):
 
         # Offset: lineno pairs, only for offsets which start line.
         # Locally we use list for more convenient iteration using indices
+        if self.version > 1.4:
             linestarts = list(self.opc.findlinestarts(code_obj))
+        else:
+            linestarts = [[0, 1]]
         self.linestarts = dict(linestarts)
 
         # 'List-map' which shows line number of current op and offset of

uncompyle6/scanners/scanner14.py

@@ -0,0 +1,36 @@
+#  Copyright (c) 2018 by Rocky Bernstein
+"""
+Python 1.4 bytecode decompiler massaging.
+
+This massages tokenized 1.4 bytecode to make it more amenable for
+grammar parsing.
+"""
+
+import uncompyle6.scanners.scanner15 as scan
+# from uncompyle6.scanners.scanner26 import ingest as  ingest26
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_14
+JUMP_OPS = opcode_14.JUMP_OPS
+
+# We base this off of 1.5 instead of the other way around
+# because we cleaned things up this way.
+# The history is that 2.7 support is the cleanest,
+# then from that we got 2.6 and so on.
+class Scanner14(scan.Scanner15):
+    def __init__(self, show_asm=False):
+        scan.Scanner15.__init__(self, show_asm)
+        self.opc = opcode_14
+        self.opname = opcode_14.opname
+        self.version = 1.4
+        self.genexpr_name = '<generator expression>'
+        return
+
+    # def ingest22(self, co, classname=None, code_objects={}, show_asm=None):
+    #     tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm)
+    #     tokens = [t for t in tokens if t.kind != 'SET_LINENO']
+
+    #     # for t in tokens:
+    #     #     print(t)
+
+        return tokens, customize

uncompyle6/scanners/scanner15.py

@@ -1,6 +1,6 @@
-#  Copyright (c) 2016-2017 by Rocky Bernstein
+#  Copyright (c) 2016-2018 by Rocky Bernstein
 """
-Python 1.5 bytecode decompiler scanner.
+Python 1.5 bytecode decompiler massaging.
 
 This massages tokenized 1.5 bytecode to make it more amenable for
 grammar parsing.

uncompyle6/scanners/scanner21.py

@@ -1,6 +1,6 @@
-#  Copyright (c) 2016-2017 by Rocky Bernstein
+#  Copyright (c) 2016-2018 by Rocky Bernstein
 """
-Python 2.1 bytecode scanner/deparser
+Python 2.1 bytecode massaging.
 
 This massages tokenized 2.1 bytecode to make it more amenable for
 grammar parsing.

uncompyle6/scanners/scanner22.py

@@ -1,6 +1,6 @@
-#  Copyright (c) 2016-2017 by Rocky Bernstein
+#  Copyright (c) 2016-2018 by Rocky Bernstein
 """
-Python 2.2 bytecode ingester.
+Python 2.2 bytecode massaging.
 
 This massages tokenized 2.2 bytecode to make it more amenable for
 grammar parsing.

uncompyle6/scanners/scanner23.py

@@ -1,6 +1,6 @@
-#  Copyright (c) 2016-2017 by Rocky Bernstein
+#  Copyright (c) 2016-2018 by Rocky Bernstein
 """
-Python 2.3 bytecode scanner/deparser
+Python 2.3 bytecode massaging.
 
 This massages tokenized 2.3 bytecode to make it more amenable for
 grammar parsing.

uncompyle6/scanners/scanner24.py

@@ -1,6 +1,6 @@
 #  Copyright (c) 2016-2017 by Rocky Bernstein
 """
-Python 2.4 bytecode scanner/deparser
+Python 2.4 bytecode massaging.
 
 This massages tokenized 2.7 bytecode to make it more amenable for
 grammar parsing.

uncompyle6/scanners/scanner25.py

@@ -1,6 +1,6 @@
-#  Copyright (c) 2015-2017 by Rocky Bernstein
+#  Copyright (c) 2015-2018 by Rocky Bernstein
 """
-Python 2.5 bytecode scanner/deparser
+Python 2.5 bytecode massaging.
 
 This overlaps Python's 2.5's dis module, but it can be run from
 Python 3 and other versions of Python. Also, we save token

uncompyle6/semantics/fragments.py

@@ -1115,7 +1115,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
                 self.p.insts = p_insts
             except (parser.ParserError(e), AssertionError(e)):
                 raise ParserError(e, tokens)
-            maybe_show_tree(self.showast, ast)
+            maybe_show_tree(self, ast)
             return ast
 
         # The bytecode for the end of the main routine has a
@@ -1154,7 +1154,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
         except (parser.ParserError(e), AssertionError(e)):
             raise ParserError(e, tokens)
 
-        maybe_show_tree(self.showast, ast)
+        maybe_show_tree(self, ast)
 
         checker(ast, False, self.ast_errors)
 

uncompyle6/semantics/make_function.py

@@ -42,7 +42,7 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
         """
         if default:
             value = self.traverse(default, indent='')
-            maybe_show_tree_param_default(self.showast, name, value)
+            maybe_show_tree_param_default(self, name, value)
             result = '%s=%s' % (name,  value)
             if result[-2:] == '= ':	# default was 'LOAD_CONST None'
                 result += 'None'

uncompyle6/semantics/pysource.py

@@ -251,6 +251,55 @@ class SourceWalker(GenericASTTraversal, object):
 
         return
 
+    def str_with_template(self, ast):
+        stream = sys.stdout
+        stream.write(self.str_with_template1(ast, '', None))
+        stream.write('\n')
+
+    def str_with_template1(self, ast, indent, sibNum=None):
+        rv = str(ast.kind)
+
+        if sibNum is not None:
+            rv = "%2d. %s" % (sibNum, rv)
+        enumerate_children = False
+        if len(ast) > 1:
+            rv += " (%d)" % (len(ast))
+            enumerate_children = True
+
+        mapping = self._get_mapping(ast)
+        table = mapping[0]
+        key = ast
+        for i in mapping[1:]:
+            key = key[i]
+            pass
+
+        if key.kind in table:
+            rv += ": %s" % str(table[key.kind])
+
+        rv = indent + rv
+        indent += '    '
+        i = 0
+        for node in ast:
+            if hasattr(node, '__repr1__'):
+                if enumerate_children:
+                    child =  self.str_with_template1(node, indent, i)
+                else:
+                    child = self.str_with_template1(node, indent, None)
+            else:
+                inst = node.format(line_prefix='L.')
+                if inst.startswith("\n"):
+                    # Nuke leading \n
+                    inst = inst[1:]
+                if enumerate_children:
+                    child = indent + "%2d. %s" % (i, inst)
+                else:
+                    child = indent + inst
+                pass
+            rv += "\n" + child
+            i += 1
+        return rv
+
+
     def indent_if_source_nl(self, line_number, indent):
         if (line_number != self.line_number):
             self.write("\n" + self.indent + INDENT_PER_LEVEL[:-1])
@@ -2083,7 +2132,7 @@ class SourceWalker(GenericASTTraversal, object):
                 raise ParserError(e, tokens)
             except AssertionError, e:
                 raise ParserError(e, tokens)
-            maybe_show_tree(self.showast, ast)
+            maybe_show_tree(self, ast)
             return ast
 
         # The bytecode for the end of the main routine has a
@@ -2116,7 +2165,7 @@ class SourceWalker(GenericASTTraversal, object):
         except python_parser.ParserError, e:
             raise ParserError(e, tokens)
 
-        maybe_show_tree(self.showast, ast)
+        maybe_show_tree(self, ast)
 
         checker(ast, False, self.ast_errors)
 

uncompyle6/show.py

@@ -35,7 +35,7 @@ def maybe_show_asm(showasm, tokens):
             stream.write('\n')
 
 
-def maybe_show_tree(show_tree, ast):
+def maybe_show_tree(walker, ast):
     """
     Show the ast based on the showast flag (or file object), writing to the
     appropriate stream depending on the type of the flag.
@@ -45,11 +45,14 @@ def maybe_show_tree(show_tree, ast):
                       like object, into which the ast will be written).
     :param ast:     The ast to show.
     """
-    if show_tree:
+    if walker.showast:
-        if hasattr(show_tree, 'write'):
+        if hasattr(walker.showast, 'write'):
-            stream = show_tree
+            stream = walker.showast
         else:
             stream = sys.stdout
+        if walker.showast == 'Full':
+            walker.str_with_template(ast)
+        else:
             stream.write(str(ast))
         stream.write('\n')
 

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.1.3'
+VERSION='3.2.0'