Merge commit '1d7a3c6444eab5a02d899f789f2a57cfdcbc5a84' into python-2.4

ChangeLog

@@ -1,3 +1,95 @@
+2017-10-10  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse24.py, uncompyle6/scanners/scanner3.py: 
+	Misc bugs
+
+2017-10-05  rocky <rb@dustyfeet.com>
+
+	* test/simple_source/branching/02_ifelse_lambda.py: One more test
+
+2017-10-05  rocky <rb@dustyfeet.com>
+
+	* .gitignore, pytest/test_grammar.py, uncompyle6/parser.py,
+	uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
+	uncompyle6/parsers/parse3.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/pysource.py: Sync with master
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse24.py,
+	uncompyle6/parsers/parse26.py: handle newer parser reduction
+	behavior
+
+2017-10-03  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Go over table-semantics
+	description yet again
+
+2017-10-02  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: Sync
+	with master
+
+2017-09-26  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/parsers/parse3.py: Annotation field can be unicode...  When deparsing Python 3.x from Python 2.
+
+2017-09-25  rocky <rb@dustyfeet.com>
+
+	* __pkginfo__.py: Require xdis 3.6.0 or greater
+
+2017-09-25  rocky <rb@dustyfeet.com>
+
+	* : commit 0654aed6c823d0bb20abdc866481ca5950db72f7 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Sep 21 11:29:17 2017 -0400
+
+2017-09-21  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/semantics/consts.py,
+	uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py: 
+	Unit test for format-specifiers
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: Tidy pysource and fragments
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/consts.py: Tidy/regularize table entry
+	formatting
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* test/test_pythonlib.py, uncompyle6/semantics/pysource.py: small
+	fixes...  test_pythonlib.py: it is sys.exit not exit pysource.py: restore node
+	type on async_call function
+
+2017-09-20  rocky <rb@dustyfeet.com>
+
+	* pytest/test_pysource.py, uncompyle6/semantics/pysource.py: Start
+	pysource unit test
+
+2017-09-17  rocky <rb@dustyfeet.com>
+
+	* uncompyle6/semantics/fragments.py,
+	uncompyle6/semantics/pysource.py: emgine -> template_engine
+
+2017-08-31  rocky <rb@dustyfeet.com>
+
+	* : commit 356ea6c7705a557cb3e725d1aca8589dd62b5cdf Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Aug 31 09:50:48 2017 -0400
+
+2017-08-31  rocky <rb@dustyfeet.com>
+
+	* : commit 4d5843851543bfb3c97fc3c49036f1a971fc1d66 Author: rocky
+	<rb@dustyfeet.com> Date:   Thu Aug 31 08:53:58 2017 -0400
+
+2017-08-15  rocky <rb@dustyfeet.com>
+
+	* Makefile: 3.7 support
+
 2017-08-15  rocky <rb@dustyfeet.com>
 
 	* : commit c54a47b15f85be50d2278aa79fd514eb08580e65 Author: rocky

NEWS

@@ -1,4 +1,4 @@
-uncompyle6 2.12.0 2017-09-25
+uncompyle6 2.12.0 2017-09-26
 
 - Use xdis 3.6.0 or greater now
 - Small semantic table cleanups

README.rst

@@ -4,7 +4,7 @@ uncompyle6
 ==========
 
 A native Python cross-version Decompiler and Fragment Decompiler.
-Follows in the tradition of decompyle, uncompyle, and uncompyle2.
+The successor to decompyle, uncompyle, and uncompyle2.
 
 
 Introduction

pytest/test_pysource.py

@@ -28,12 +28,16 @@ from uncompyle6.semantics.fragments import (
     TABLE_DIRECT_FRAGMENT,
     )
 
+skip_for_now = "DELETE_DEREF".split()
+
 def test_tables():
     for t, name, fragment in (
             (TABLE_DIRECT, 'TABLE_DIRECT', False),
             (TABLE_R, 'TABLE_R', False),
             (TABLE_DIRECT_FRAGMENT, 'TABLE_DIRECT_FRAGMENT', True)):
         for k, entry in t.iteritems():
+            if k in skip_for_now:
+                continue
             fmt = entry[0]
             arg = 1
             i = 0

test/Makefile

@@ -39,7 +39,7 @@ check-3.3: check-bytecode
 
 #: Run working tests from Python 3.4
 check-3.4: check-bytecode check-3.4-ok check-2.7-ok
-	$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
+	$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
 
 #: Run working tests from Python 3.5
 check-3.5: check-bytecode

uncompyle6/parser.py

@@ -42,21 +42,25 @@ class PythonParser(GenericASTBuilder):
         else:
             return self.ast_first_offset(ast[0])
 
-    def add_unique_rule(self, rule, opname, count, customize):
+    def add_unique_rule(self, rule, opname, arg_count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
-           opname and count are used in the customize() semantic the actions
+           opname and stack_count are used in the customize() semantic
-           to add the semantic action rule. Often, count is not used.
+           the actions to add the semantic action rule. Stack_count is
+           used in custom opcodes like MAKE_FUNCTION to indicate how
+           many arguments it has. Often it is not used.
         """
         if rule not in self.new_rules:
             # print("XXX ", rule) # debug
             self.new_rules.add(rule)
             self.addRule(rule, nop_func)
-            customize[opname] = count
+            customize[opname] = arg_count
             pass
         return
 
     def add_unique_rules(self, rules, customize):
-        """Add rules (a list of string) to grammar
+        """Add rules (a list of string) to grammar. Note that
+        the rules must not be those that set arg_count in the
+        custom dictionary.
         """
         for rule in rules:
             if len(rule) == 0:
@@ -66,7 +70,9 @@ class PythonParser(GenericASTBuilder):
         return
 
     def add_unique_doc_rules(self, rules_str, customize):
-        """Add rules (a docstring-like list of rules) to grammar
+        """Add rules (a docstring-like list of rules) to grammar.
+        Note that the rules must not be those that set arg_count in the
+        custom dictionary.
         """
         rules = [r.strip() for r in rules_str.split("\n")]
         self.add_unique_rules(rules, customize)

uncompyle6/scanner.py

@@ -20,7 +20,7 @@ from xdis.magics import py_str2float
 # The byte code versions we support
 PYTHON_VERSIONS = (1.5,
                    2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7,
-                   3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6)
+                   3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7)
 
 # FIXME: DRY
 if PYTHON3:

uncompyle6/scanners/scanner2.py

@@ -99,12 +99,18 @@ class Scanner2(Scanner):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # Container for tokens
+        # list of tokens/instructions
         tokens = []
 
+        # "customize" is a dict whose keys are nonterminals
+        # and the value is the argument stack entries for that
+        # nonterminal. The count is a little hoaky. It is mostly
+        # not used, but sometimes it is.
+        # "customize" is a dict whose keys are nonterminals
         customize = {}
+
         if self.is_pypy:
-            customize['PyPy'] = 1
+            customize['PyPy'] = 0
 
         Token = self.Token # shortcut
 

uncompyle6/scanners/scanner3.py

@@ -175,12 +175,16 @@ class Scanner3(Scanner):
             for instr in bytecode.get_instructions(co):
                 print(instr._disassemble())
 
-        # Container for tokens
+        # list of tokens/instructions
         tokens = []
 
+        # "customize" is a dict whose keys are nonterminals
+        # and the value is the argument stack entries for that
+        # nonterminal. The count is a little hoaky. It is mostly
+        # not used, but sometimes it is.
         customize = {}
         if self.is_pypy:
-            customize['PyPy'] = 1
+            customize['PyPy'] = 0
 
         self.code = array('B', co.co_code)
         self.build_lines_data(co)
@@ -336,7 +340,7 @@ class Scanner3(Scanner):
                     attr = (pos_args, name_pair_args, annotate_args)
                 tokens.append(
                     Token(
-                        type_ = opname,
+                        opname = opname,
                         attr = attr,
                         pattr = pattr,
                         offset = inst.offset,
@@ -414,7 +418,7 @@ class Scanner3(Scanner):
             last_op_was_break = opname == 'BREAK_LOOP'
             tokens.append(
                 Token(
-                    type_ = opname,
+                    opname = opname,
                     attr = argval,
                     pattr = pattr,
                     offset = inst.offset,

uncompyle6/scanners/tok.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016 by Rocky Bernstein
+#  Copyright (c) 2016-2017 by Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
 
@@ -8,7 +8,7 @@ from uncompyle6 import PYTHON3
 if PYTHON3:
     intern = sys.intern
 
-class Token:
+class Token():
     """
     Class representing a byte-code instruction.
 
@@ -16,13 +16,12 @@ class Token:
     the contents of one line as output by dis.dis().
     """
     # FIXME: match Python 3.4's terms:
-    #    type_ should be opname
     #    linestart = starts_line
     #    attr = argval
     #    pattr = argrepr
-    def __init__(self, type_, attr=None, pattr=None, offset=-1,
+    def __init__(self, opname, attr=None, pattr=None, offset=-1,
                  linestart=None, op=None, has_arg=None, opc=None):
-        self.type = intern(type_)
+        self.type = intern(opname)
         self.op = op
         self.has_arg = has_arg
         self.attr = attr

uncompyle6/semantics/consts.py

@@ -173,12 +173,7 @@ TABLE_DIRECT = {
     'ret_cond':         ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ),
     'conditionalnot':   ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27) ),
     'ret_cond_not':     ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ),
-    'conditional_lambda':  ( '%c if %c else %c', 2, 0, 4),
+    'conditional_lambda':  ( '(%c if %c else %c)', 2, 0, 3),
-
-    # The semicolon is because Python 3.x can have be dead code as a result of its
-    # optimization. We don't Python's remove dead code (yet) anymore than Python does.
-    # So without that we would have "return 2return3" rather than "return 2;return 3"
-    'return_lambda':    ('return %c;', 0),
 
     'compare':		    ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
     'cmp_list':		    ( '%p %p', (0, 29), (1, 30)),

uncompyle6/semantics/pysource.py

@@ -43,29 +43,33 @@ Python.
 # We allow for a couple of ways to interact with a node in a tree.  So
 # step 1 after not seeing a custom method for a nonterminal is to
 # determine from what point of view tree-wise the rule is applied.
-# In the diagram below, "N" is a nonterminal name, and K is the table
+
-# key name; we show where those are with respect to each other in the
+# In the diagram below, N is a nonterminal name, and K also a nonterminal
+# name but the one used as a key in the table.
+# we show where those are with respect to each other in the
 # AST tree for N.
 #
 #
-#          N                  N               N&K
+#          N&K               N                  N
-#        / | ... \          / | ... \        / | ... \
+#         / | ... \        / | ... \          / | ... \
-#       O  O      O        O  O      K      O  O      O
+#        O  O      O      O  O      K         O O      O
-#                 |
+#                                                      |
-#                 K
+#                                                      K
+#      TABLE_DIRECT      TABLE_R             TABLE_R0
 #
-#   MAP_R0 (TABLE_R0)  MAP_R (TABLE_R)  MAP_DIRECT (TABLE_DIRECT)
+#   The default table is TABLE_DIRECT mapping By far, most rules used work this way.
-#
-#   The default is a "TABLE_DIRECT" mapping By far, most rules used work this way.
 #   TABLE_R0 is rarely used.
 #
-#   The key K is then extracted from the
+#   The key K is then extracted from the subtree and used to find one
-#   subtree and used to find a table entry T[K], if any.  The result is a
+#   of the tables, T listed above.  The result after applying T[K] is
-#   format string and arguments (a la printf()) for the formatting engine.
+#   a format string and arguments (a la printf()) for the formatting
+#   engine.
+#
 #   Escapes in the format string are:
 #
 #     %c  evaluate the node recursively. Its argument is a single
 #         integer representing a node index.
+#
 #     %p  like %c but sets the operator precedence.
 #         Its argument then is a tuple indicating the node
 #         index and the precidence value, an integer.
@@ -1861,10 +1865,9 @@ class SourceWalker(GenericASTTraversal, object):
                     node[0].attr == 1):
                     self.write(',')
             elif typ == 'c':
-                if isinstance(entry[arg], int):
+                entry_node = node[entry[arg]]
-                    entry_node = node[entry[arg]]
+                self.preorder(entry_node)
-                    self.preorder(entry_node)
+                arg += 1
-                    arg += 1
             elif typ == 'p':
                 p = self.prec
                 (index, self.prec) = entry[arg]