Merge branch 'master' into python-2.4

and it shouldn't have

NEWS

@@ -1,3 +1,7 @@
+uncompyle6 3.2.1 2018-06-04 MF
+
+- Python 1.4 and 1.5 bug fixes
+
 uncompyle6 3.2.0 2018-05-19 Rocket Scientist
 
 - Add rudimentary 1.4 support (still a bit buggy)

appveyor.yml

@@ -47,7 +47,7 @@ install:
 
   # Upgrade to the latest version of pip to avoid it displaying warnings
   # about it being out of date.
-  - "pip install --disable-pip-version-check --user --upgrade pip"
+  - "%PYTHON%\\python.exe -m pip install --disable-pip-version-check --user --upgrade pip"
 
   # Install the build dependencies of the project. If some dependencies contain
   # compiled extensions and are not provided as pre-built wheel packages,

test/Makefile

@@ -274,7 +274,7 @@ check-3.4-ok:
 2.6:
 
 #: PyPy 5.0.x with Python 2.7 ...
-pypy-2.7 5.0 5.3:
+pypy-2.7 5.0 5.3 6.0:
 	$(PYTHON) test_pythonlib.py --bytecode-pypy2.7 --verify
 
 #: PyPy 2.4.x with Python 3.2 ...

test/simple_source/bug14/00_unpack_list.py

@@ -0,0 +1,3 @@
+# Python 1.4 tzparse.py, but also appears in 1.5
+
+[tzname, delta] = __file__

test/simple_source/bug14/01_print.py

@@ -0,0 +1,6 @@
+# Python 1.4 aifc.py
+# Something weird about the final "print" and PRINT_NL_CONT followed by PRINT_NL
+def _readmark(self, markers):
+    if self._markers: print 'marker',
+    else: print 'markers',
+    print 'instead of', markers

test/simple_source/bug14/02_continue.py

@@ -0,0 +1,11 @@
+# Python 1.4 cgi.py
+# Bug was in "continue" detection.
+# 1.4 doesn't have lnotab and our CONTINUE detection is off.
+def parse_multipart(params, pdict):
+    while params:
+	if params.has_key('name'):
+	   params = None
+	else:
+	    continue
+
+    return None

test/stdlib/compile-file-1x.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""byte compiles a Python 1.x program"""
+import sys
+if len(sys.argv) != 2:
+    print("Usage: compile-file.py *python-file*")
+    sys.exit(1)
+source = sys.argv[1]
+
+# assert source.endswith('.py')
+basename = source[:-3]
+
+# We do this crazy way to support Python 1.4 which
+# doesn't support version_info.
+PY_VERSION = sys.version[:3]
+
+bytecode = "%s-%s.pyc" % (basename, PY_VERSION)
+
+import py_compile
+print("# compiling %s to %s" % (source, bytecode))
+py_compile.compile(source, bytecode)
+# import os
+# os.system("../bin/uncompyle6 %s" % bytecode)

test/stdlib/compile-file.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+"""byte compiles a Python program after version 2.2 or so. Also see compile_file_1x.py"""
 import sys
 if len(sys.argv) != 2:
     print("Usage: compile-file.py *python-file*")

uncompyle6/parser.py

@@ -618,18 +618,25 @@ def get_python_parser(
     # a lazy way of doing the import?
 
     if version < 3.0:
-        if version == 1.5:
-            import uncompyle6.parsers.parse15 as parse15
-            if compile_mode == 'exec':
-                p = parse15.Python15Parser(debug_parser)
-            else:
-                p = parse15.Python15ParserSingle(debug_parser)
-        elif version == 2.1:
-            import uncompyle6.parsers.parse21 as parse21
-            if compile_mode == 'exec':
-                p = parse21.Python21Parser(debug_parser)
-            else:
-                p = parse21.Python21ParserSingle(debug_parser)
+        if version < 2.2:
+            if version == 1.4:
+                import uncompyle6.parsers.parse14 as parse14
+                if compile_mode == 'exec':
+                    p = parse14.Python14Parser(debug_parser)
+                else:
+                    p = parse14.Python14ParserSingle(debug_parser)
+            elif version == 1.5:
+                import uncompyle6.parsers.parse15 as parse15
+                if compile_mode == 'exec':
+                    p = parse15.Python15Parser(debug_parser)
+                else:
+                    p = parse15.Python15ParserSingle(debug_parser)
+            elif version == 2.1:
+                import uncompyle6.parsers.parse21 as parse21
+                if compile_mode == 'exec':
+                    p = parse21.Python21Parser(debug_parser)
+                else:
+                    p = parse21.Python21ParserSingle(debug_parser)
         elif version == 2.2:
             import uncompyle6.parsers.parse22 as parse22
             if compile_mode == 'exec':

uncompyle6/parsers/parse14.py

@@ -11,12 +11,51 @@ class Python14Parser(Python15Parser):
         # Nothing here yet, but will need to add UNARY_CALL, BINARY_CALL,
         # RAISE_EXCEPTION, BUILD_FUNCTION, UNPACK_ARG, UNPACK_VARARG, LOAD_LOCAL,
         # SET_FUNC_ARGS, and RESERVE_FAST
+
+        # FIXME: should check that this indeed around __doc__
+        # Possibly not strictly needed
+        stmt     ::= doc_junk
+        doc_junk ::= LOAD_CONST POP_TOP
+
+        # Not sure why later Python's omit the COME_FROM
+        jb_pop14  ::= JUMP_BACK COME_FROM POP_TOP
+
+        whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt
+                          jb_pop14
+                          POP_BLOCK else_suitel COME_FROM
+
+        print_items_nl_stmt ::= expr PRINT_ITEM_CONT print_items_opt PRINT_NEWLINE_CONT
+
+
+        # 1.4 doesn't have linenotab, and although this shouldn't
+        # be a show stopper, our CONTINUE detection is off here.
+        continue ::= JUMP_BACK
         """
 
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         super(Python14Parser, self).__init__(debug_parser)
         self.customized = {}
 
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python14Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules("""
+        whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt
+                          jb_pop
+                          POP_BLOCK else_suitel COME_FROM
+        """)
+        self.check_reduce['doc_junk'] = 'tokens'
+
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        invalid = super(Python14Parser,
+                        self).reduce_is_invalid(rule, ast,
+                                                tokens, first, last)
+        if invalid or tokens is None:
+            return invalid
+        if rule[0] == 'doc_junk':
+            return not isinstance(tokens[first].pattr, str)
+
+
 
 class Python14ParserSingle(Python14Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse15.py

@@ -2,7 +2,7 @@
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
-from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse21 import Python21Parser
 
 class Python15Parser(Python21Parser):
@@ -23,6 +23,17 @@ class Python15Parser(Python21Parser):
         importlist  ::= IMPORT_FROM
         """
 
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python15Parser, self).customize_grammar_rules(tokens, customize)
+        for i, token in enumerate(tokens):
+            opname = token.kind
+            opname_base = opname[:opname.rfind('_')]
+
+            if opname_base == 'UNPACK_LIST':
+                self.addRule("store ::= unpack_list", nop_func)
+
+
+
 class Python15ParserSingle(Python15Parser, PythonParserSingle):
     pass
 

uncompyle6/parsers/parse2.py

@@ -292,7 +292,6 @@ class Python2Parser(PythonParser):
 
             # The order of opname listed is roughly sorted below
             if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
-                v = token.attr
                 collection = opname_base[opname_base.find('_')+1:].lower()
                 rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname)
                 self.add_unique_rules([
@@ -393,7 +392,6 @@ class Python2Parser(PythonParser):
                     """, nop_func)
                 continue
             elif opname == 'JUMP_IF_NOT_DEBUG':
-                v = token.attr
                 self.addRule("""
                     jmp_true_false ::= POP_JUMP_IF_TRUE
                     jmp_true_false ::= POP_JUMP_IF_FALSE

uncompyle6/parsers/parse26.py

@@ -349,7 +349,7 @@ class Python26Parser(Python2Parser):
             # For now, we won't let the 2nd 'expr' be a "conditional_not"
             # However in < 2.6 where we don't have if/else expression it *can*
             # be.
-            if ast[2][0] == 'conditional_not':
+            if self.version >= 2.6 and ast[2][0] == 'conditional_not':
                 return True
 
             test_index = last

uncompyle6/scanners/scanner14.py

@@ -32,5 +32,5 @@ class Scanner14(scan.Scanner15):
 
     #     # for t in tokens:
     #     #     print(t)
-
-        return tokens, customize
+    #
+    #   return tokens, customize

uncompyle6/scanners/scanner15.py

@@ -25,3 +25,17 @@ class Scanner15(scan.Scanner21):
         self.version = 1.5
         self.genexpr_name = '<generator expression>'
         return
+
+    def ingest(self, co, classname=None, code_objects={}, show_asm=None):
+        """
+        Pick out tokens from an uncompyle6 code object, and transform them,
+        returning a list of uncompyle6 Token's.
+
+        The transformations are made to assist the deparsing grammar.
+        """
+        tokens, customize = scan.Scanner21.ingest(self, co, classname, code_objects, show_asm)
+        for t in tokens:
+            if t.op == self.opc.UNPACK_LIST:
+                t.kind = 'UNPACK_LIST_%d' % t.attr
+            pass
+        return tokens, customize

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.2.0'
+VERSION='3.2.1'