Merge branch 'master' into python-2.4

NEWS

@@ -1,3 +1,8 @@
+uncompyle6 3.2.3 2018-06-04 Michael Cohen flips and Fleetwood Redux
+
+- Python 1.3 support 3.0 bug and
+- fix botched parameter ordering of 3.x in last release
+
 uncompyle6 3.2.2 2018-06-04 When I'm 64
 
 - Python 3.0 support and bug fixes

README.rst

@@ -11,8 +11,9 @@ Introduction
 ------------
 
 *uncompyle6* translates Python bytecode back into equivalent Python
-source code. It accepts bytecodes from Python version 1.5, and 2.1 to
+source code. It accepts bytecodes from Python version 1.3 to version
-3.7 or so, including PyPy bytecode and Dropbox's Python 2.5 bytecode.
+3.7, spanning over 22 years of Python releases. We include Dropbox's
+Python 2.5 bytecode and some PyPy bytecode.
 
 Why this?
 ---------
@@ -75,7 +76,7 @@ Requirements
 The code here can be run on Python versions 2.6 or later, PyPy 3-2.4,
 or PyPy-5.0.1.  Python versions 2.4-2.7 are supported in the
 python-2.4 branch.  The bytecode files it can read have been tested on
-Python bytecodes from versions 1.5, 2.1-2.7, and 3.0-3.6 and the
+Python bytecodes from versions 1.4, 2.1-2.7, and 3.0-3.6 and the
 above-mentioned PyPy versions.
 
 Installation
@@ -217,7 +218,7 @@ See Also
 * https://github.com/zrax/pycdc : purports to support all versions of Python. It is written in C++ and is most accurate for Python versions around 2.7 and 3.3 when the code was more actively developed. Accuracy for more recent versions of Python 3 and early versions of Python are especially lacking. See its `issue tracker <https://github.com/zrax/pycdc/issues>`_ for details. Currently lightly maintained.
 * https://code.google.com/archive/p/unpyc3/ : supports Python 3.2 only. The above projects use a different decompiling technique than what is used here. Currently unmaintained.
 * https://github.com/figment/unpyc3/ : fork of above, but supports Python 3.3 only. Includes some fixes like supporting function annotations. Currently unmaintained.
-* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. Because of it specificity it can sometimes to better than uncompyle6 which we can't do withouth breaking other 2.7 cases. Currently lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
+* https://github.com/wibiti/uncompyle2 : supports Python 2.7 only, but does that fairly well. There situtations where `uncompyle6` results are incorrect while `uncompyle2` results are not, but more often uncompyle6 is correct when uncompyle2 is not. Because `uncompyle6` adheres to accuracy over idiomatic Python, `uncompyle2` can produce more natural-looking code when it is correct. Currently `uncompyle2` is lightly maintained. See its issue `tracker <https://github.com/wibiti/uncompyle2/issues>`_ for more details
 * `How to report a bug <https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md>`_
 * The HISTORY_ file.
 * https://github.com/rocky/python-xdis : Cross Python version disassembler

test/Makefile

@@ -22,7 +22,7 @@ COVER_DIR=../tmp/grammar-cover
 # Run short tests
 check-short:
 	@$(PYTHON) -V && PYTHON_VERSION=`$(PYTHON) -V 2>&1 | cut -d ' ' -f 2 | cut -d'.' -f1,2`; \
-	$(MAKE) check-bytecode
+	$(MAKE) check-bytecode-short
 
 # Run all tests
 check:
@@ -91,16 +91,31 @@ check-bytecode-3:
        --bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
        --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
 
-#: Check deparsing bytecode that works running Python 2 and Python 3
+#: Check deparsing on selected bytecode 3.x
+check-bytecode-3-short:
+	$(PYTHON) test_pythonlib.py \
+       --bytecode-3.4 --bytecode-3.5 --bytecode-3.6
+
+#: Check deparsing bytecode on all Python 2 and Python 3 versions
 check-bytecode: check-bytecode-3
 	$(PYTHON) test_pythonlib.py \
-	--bytecode-1.4 --bytecode-1.5 \
+	--bytecode-1.3 --bytecode-1.4 --bytecode-1.5 \
         --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
 	--bytecode-2.1 --bytecode-2.2 --bytecode-2.3 --bytecode-2.4 \
         --bytecode-2.5 --bytecode-2.6 --bytecode-2.7 \
         --bytecode-pypy2.7
 
 
+#: Check deparsing bytecode on selected Python 2 and Python 3 versions
+check-bytecode-short: check-bytecode-3-short
+	$(PYTHON) test_pythonlib.py \
+        --bytecode-2.6 --bytecode-2.7 --bytecode-pypy2.7
+
+
+#: Check deparsing bytecode 1.3 only
+check-bytecode-1.3:
+	$(PYTHON) test_pythonlib.py --bytecode-1.3
+
 #: Check deparsing bytecode 1.4 only
 check-bytecode-1.4:
 	$(PYTHON) test_pythonlib.py --bytecode-1.4
@@ -210,6 +225,11 @@ check-bytecode-2.7:
 check-bytecode-3.0:
 	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
 
+#: Check deparsing Python 3.0
+check-bytecode-3.0:
+	$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify
+        # $(PYTHON) test_pythonlib.py --bytecode-3.0-run --verify-run
+
 #: Check deparsing Python 3.1
 check-bytecode-3.1:
 	$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify

test/test_pythonlib.py

@@ -76,7 +76,7 @@ for  vers in (2.7, 3.4, 3.5, 3.6):
     test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers)
     pass
 
-for  vers in (1.4, 1.5,
+for  vers in (1.3, 1.4, 1.5,
               2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
               3.0, 3.1, 3.2, 3.3,
               3.4, 3.5, 3.6, 3.7, 'pypy3.2', 'pypy2.7'):

uncompyle6/parser.py

@@ -619,7 +619,13 @@ def get_python_parser(
 
     if version < 3.0:
         if version < 2.2:
-            if version == 1.4:
+            if version == 1.3:
+                import uncompyle6.parsers.parse13 as parse13
+                if compile_mode == 'exec':
+                    p = parse13.Python14Parser(debug_parser)
+                else:
+                    p = parse13.Python14ParserSingle(debug_parser)
+            elif version == 1.4:
                 import uncompyle6.parsers.parse14 as parse14
                 if compile_mode == 'exec':
                     p = parse14.Python14Parser(debug_parser)

uncompyle6/parsers/parse13.py

@@ -0,0 +1,49 @@
+#  Copyright (c) 2018 Rocky Bernstein
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.parser import PythonParserSingle
+from uncompyle6.parsers.parse14 import Python14Parser
+
+class Python13Parser(Python14Parser):
+
+    def p_misc13(self, args):
+        """
+        # Nothing here yet, but will need to add LOAD_GLOBALS
+        """
+
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
+        super(Python13Parser, self).__init__(debug_parser)
+        self.customized = {}
+
+    # def customize_grammar_rules(self, tokens, customize):
+    #     super(Python13Parser, self).customize_grammar_rules(tokens, customize)
+    #     self.remove_rules("""
+    #     whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt
+    #                       jb_pop
+    #                       POP_BLOCK else_suitel COME_FROM
+    #     """)
+    #     self.check_reduce['doc_junk'] = 'tokens'
+
+
+    # def reduce_is_invalid(self, rule, ast, tokens, first, last):
+    #     invalid = super(Python14Parser,
+    #                     self).reduce_is_invalid(rule, ast,
+    #                                             tokens, first, last)
+    #     if invalid or tokens is None:
+    #         return invalid
+    #     if rule[0] == 'doc_junk':
+    #         return not isinstance(tokens[first].pattr, str)
+
+
+
+class Python13ParserSingle(Python13Parser, PythonParserSingle):
+    pass
+
+if __name__ == '__main__':
+    # Check grammar
+    p = Python13Parser()
+    p.check_grammar()
+    p.dump_grammar()
+
+# local variables:
+# tab-width: 4

uncompyle6/parsers/parse14.py

@@ -8,12 +8,11 @@ class Python14Parser(Python15Parser):
 
     def p_misc14(self, args):
         """
-        # Nothing here yet, but will need to add UNARY_CALL, BINARY_CALL,
+        # Not much here yet, but will probably need to add UNARY_CALL, BINARY_CALL,
         # RAISE_EXCEPTION, BUILD_FUNCTION, UNPACK_ARG, UNPACK_VARARG, LOAD_LOCAL,
         # SET_FUNC_ARGS, and RESERVE_FAST
 
-        # FIXME: should check that this indeed around __doc__
+        # Not strictly needed, but tidies up output
-        # Possibly not strictly needed
         stmt     ::= doc_junk
         doc_junk ::= LOAD_CONST POP_TOP
 

uncompyle6/parsers/parse30.py

@@ -73,6 +73,8 @@ class Python30Parser(Python31Parser):
         for_block      ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
         except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
                            POP_TOP END_FINALLY come_froms
+        except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
+                           POP_TOP END_FINALLY
         return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
         and            ::= expr JUMP_IF_FALSE POP_TOP expr COME_FROM
         whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt
@@ -93,6 +95,8 @@ class Python30Parser(Python31Parser):
         assert             ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
         return_if_lambda   ::= RETURN_END_IF_LAMBDA
         compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP compare_chained2 COME_FROM
+        except_handler     ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY
+
         """)
 
         return

uncompyle6/scanner.py

@@ -39,7 +39,7 @@ else:
 
 # The byte code versions we support.
 # Note: these all have to be floats
-PYTHON_VERSIONS = frozenset((1.4, 1.5,
+PYTHON_VERSIONS = frozenset((1.3, 1.4, 1.5,
                              2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
                              3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7))
 

uncompyle6/scanners/scanner13.py

@@ -0,0 +1,35 @@
+#  Copyright (c) 2018 by Rocky Bernstein
+"""
+Python 1.3 bytecode decompiler massaging.
+
+This massages tokenized 1.3 bytecode to make it more amenable for
+grammar parsing.
+"""
+
+import uncompyle6.scanners.scanner14 as scan
+# from uncompyle6.scanners.scanner26 import ingest as  ingest26
+
+# bytecode verification, verify(), uses JUMP_OPs from here
+from xdis.opcodes import opcode_13
+JUMP_OPS = opcode_13.JUMP_OPS
+
+# We base this off of 1.4 instead of the other way around
+# because we cleaned things up this way.
+# The history is that 2.7 support is the cleanest,
+# then from that we got 2.6 and so on.
+class Scanner13(scan.Scanner14):
+    def __init__(self, show_asm=False):
+        scan.Scanner14.__init__(self, show_asm)
+        self.opc = opcode_13
+        self.opname = opcode_13.opname
+        self.version = 1.3
+        return
+
+    # def ingest22(self, co, classname=None, code_objects={}, show_asm=None):
+    #     tokens, customize = self.parent_ingest(co, classname, code_objects, show_asm)
+    #     tokens = [t for t in tokens if t.kind != 'SET_LINENO']
+
+    #     # for t in tokens:
+    #     #     print(t)
+    #
+    #   return tokens, customize

uncompyle6/semantics/make_function.py

@@ -595,9 +595,8 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     paramnames = list(scanner_code.co_varnames[:argc])
 
     # defaults are for last n parameters, thus reverse
-    if self.version < 3.6:
+    paramnames.reverse();
-        paramnames.reverse();
+    defparams.reverse()
-        defparams.reverse()
 
     try:
         ast = self.build_ast(scanner_code._tokens,
@@ -626,8 +625,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
     else:
         params = paramnames
 
-    if not 3.1 <= self.version < 3.6:
+    params.reverse() # back to correct order
-        params.reverse() # back to correct order
 
         if code_has_star_arg(code):
             if self.version > 3.0:

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.2.2'
+VERSION='3.2.3'