Merge branch 'master' into python-2.4

__pkginfo__.py

@@ -39,7 +39,7 @@ entry_points       = {
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.7.1, < 1.8.0',
+install_requires   = ['spark-parser >= 1.8.0, < 1.9.0',
                       'xdis >= 3.6.1, < 3.7.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'

admin-tools/setup-master.sh

@@ -16,7 +16,7 @@ fi
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
-(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && \
+(cd ../python-spark && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && \
+    (cd ../python-xdis && git checkout master && pyenv local $PYTHON_VERSION) && git pull && \
-    git checkout master && pyenv local $PYTHON_VERSION
+    git checkout master && pyenv local $PYTHON_VERSION && git pull
 cd $owd

pytest/test_grammar.py

@@ -14,7 +14,8 @@ def test_grammar():
             "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dump_grammar())
 
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
-    lhs, rhs, tokens, right_recursive = p.check_sets()
+    (lhs, rhs, tokens,
+     right_recursive, dup_rhs) = p.check_sets()
     expect_lhs = set(['expr1024', 'pos_arg'])
     unused_rhs = set(['build_list', 'call_function', 'mkfunc',
                       'mklambda',
@@ -37,6 +38,15 @@ def test_grammar():
     assert expect_lhs == set(lhs)
     assert unused_rhs == set(rhs)
     assert expect_right_recursive == right_recursive
+
+    expect_dup_rhs = frozenset([('COME_FROM',), ('CONTINUE',), ('JUMP_ABSOLUTE',),
+                                ('LOAD_CONST',),
+                                ('JUMP_BACK',), ('JUMP_FORWARD',)])
+    reduced_dup_rhs = {k: dup_rhs[k] for k in dup_rhs if k not in expect_dup_rhs}
+    for k in reduced_dup_rhs:
+        print(k, reduced_dup_rhs[k])
+    # assert not reduced_dup_rhs, reduced_dup_rhs
+
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
     ignore_set = set(
             """

test/Makefile

@@ -123,7 +123,7 @@ grammar-coverage-2.7:
 	SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
 
 #: Check deparsing Python 2.6
-check-bytecode-2.6:
+pcheck-bytecode-2.6:
 	$(PYTHON) test_pythonlib.py --bytecode-2.6 --weak-verify
 
 #: Check deparsing Python 2.7

uncompyle6/parser.py

@@ -405,7 +405,6 @@ class PythonParser(GenericASTBuilder):
         importlist ::= importlist import_as
         importlist ::= import_as
         import_as  ::= IMPORT_NAME designator
-        import_as  ::= IMPORT_NAME load_attrs designator
         import_as  ::= IMPORT_FROM designator
 
         importstmt ::= LOAD_CONST LOAD_CONST import_as
@@ -413,12 +412,8 @@ class PythonParser(GenericASTBuilder):
         importfrom ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist POP_TOP
         importmultiple ::= LOAD_CONST LOAD_CONST import_as imports_cont
 
-        imports_cont ::= imports_cont import_cont
+        imports_cont ::= import_cont+
-        imports_cont ::= import_cont
+        import_cont  ::= LOAD_CONST LOAD_CONST import_as
-        import_cont ::= LOAD_CONST LOAD_CONST import_as_cont
-        import_as_cont ::= IMPORT_FROM designator
-
-        load_attrs ::= LOAD_ATTR+
         """
 
     def p_list_comprehension(self, args):

uncompyle6/parsers/parse2.py

@@ -254,19 +254,25 @@ class Python2Parser(PythonParser):
 
         PyPy adds custom rules here as well
         """
-        for opname, v in list(customize.items()):
+        if 'PyPy' in customize:
-            opname_base = opname[:opname.rfind('_')]
+            # PyPy-specific customizations
-            if opname == 'PyPy':
+            self.addRule("""
-                self.addRule("""
+                        stmt ::= assign3_pypy
-                    stmt ::= assign3_pypy
+                        stmt ::= assign2_pypy
-                    stmt ::= assign2_pypy
+                        assign3_pypy ::= expr expr expr designator designator designator
-                    assign3_pypy ::= expr expr expr designator designator designator
+                        assign2_pypy ::= expr expr designator designator
-                    assign2_pypy ::= expr expr designator designator
+                        list_compr ::= expr  BUILD_LIST_FROM_ARG _for designator list_iter
-                    list_compr ::= expr  BUILD_LIST_FROM_ARG _for designator list_iter
+                                             JUMP_BACK
-                                         JUMP_BACK
+                        """, nop_func)
-                """, nop_func)
+        for i, token in enumerate(tokens):
+            opname = token.kind
+            # FIXME: remove the "v" thing in the code below
+            if opname in customize:
+                v = customize[opname]
+            else:
                 continue
-            elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
+            opname_base = opname[:opname.rfind('_')]
+            if opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 thousands = (v//1024)
                 thirty32s = ((v//32) % 32)
                 if thirty32s > 0:
@@ -346,14 +352,16 @@ class Python2Parser(PythonParser):
                 # no longer need to add a rule
                 continue
             elif opname_base == 'MAKE_FUNCTION':
-                self.addRule('mklambda ::= %s LOAD_LAMBDA %s' %
+                if i > 0 and tokens[i-1] == 'LOAD_LAMBDA':
-                      ('pos_arg '*v, opname), nop_func)
+                    self.addRule('mklambda ::= %s LOAD_LAMBDA %s' %
+                                 ('pos_arg '*v, opname), nop_func)
                 rule = 'mkfunc ::= %s LOAD_CONST %s' % ('expr '*v, opname)
             elif opname_base == 'MAKE_CLOSURE':
                 # FIXME: use add_unique_rules to tidy this up.
+                if i > 0 and tokens[i-1] == 'LOAD_LAMBDA':
+                    self.addRule('mklambda ::= %s load_closure LOAD_LAMBDA %s' %
+                                 ('expr '*v, opname),  nop_func)
                 self.add_unique_rules([
-                    ('mklambda ::= %s load_closure LOAD_LAMBDA %s' %
-                     ('expr '*v, opname)),
                     ('genexpr ::= %s load_closure LOAD_GENEXPR %s expr'
                      ' GET_ITER CALL_FUNCTION_1' %
                      ('expr '*v, opname)),

uncompyle6/parsers/parse24.py

@@ -31,7 +31,10 @@ class Python24Parser(Python25Parser):
         importstar ::= filler LOAD_CONST IMPORT_NAME IMPORT_STAR
 
         importmultiple ::= filler LOAD_CONST import_as imports_cont
-        import_cont    ::= filler LOAD_CONST import_as_cont
+        import_cont    ::= filler LOAD_CONST import_as
+
+        import_as  ::= IMPORT_NAME load_attrs designator
+        load_attrs ::= LOAD_ATTR+
 
         # Python 2.5+ omits POP_TOP POP_BLOCK
         while1stmt ::= SETUP_LOOP l_stmts JUMP_BACK POP_TOP POP_BLOCK COME_FROM

uncompyle6/parsers/parse3.py

@@ -626,20 +626,20 @@ class Python3Parser(PythonParser):
         # Loop over instructions adding custom grammar rules based on
         # a specific instruction seen.
 
+        if 'PyPy' in customize:
+            self.addRule("""
+              stmt ::= assign3_pypy
+              stmt ::= assign2_pypy
+              assign3_pypy ::= expr expr expr designator designator designator
+              assign2_pypy ::= expr expr designator designator
+              """, nop_func)
+
         for i, token in enumerate(tokens):
             opname = token.kind
             opname_base = opname[:opname.rfind('_')]
 
-            # The order listed is roughly alphabetic by instruction opcode.
+            # The order of opname listed is roughly sorted below
-            if opname == 'PyPy':
+            if opname_base == 'BUILD_CONST_KEY_MAP':
-                self.addRule("""
-                    stmt ::= assign3_pypy
-                    stmt ::= assign2_pypy
-                    assign3_pypy ::= expr expr expr designator designator designator
-                    assign2_pypy ::= expr expr designator designator
-                """, nop_func)
-                continue
-            elif opname_base == 'BUILD_CONST_KEY_MAP':
                 # This is in 3.6+
                 kvlist_n = 'expr ' * (token.attr)
                 rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname)
@@ -695,12 +695,23 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
                 v = token.attr
-                rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
+
-                        'expr32 ' * int((v//32) % 32) +
+                is_LOAD_CLOSURE = False
-                        'expr ' * (v % 32) + opname)
-                self.add_unique_rule(rule, opname, token.attr, customize)
                 if opname_base == 'BUILD_TUPLE':
-                    rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
+                    # If is part of a "load_closure", then it is not part of a
+                    # "build_list".
+                    is_LOAD_CLOSURE = True
+                    for j in range(v):
+                        if tokens[i-j-1].kind != 'LOAD_CLOSURE':
+                            is_LOAD_CLOSURE = False
+                            break
+                    if is_LOAD_CLOSURE:
+                        rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+                if not is_LOAD_CLOSURE or v == 0:
+                    rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
+                                'expr32 ' * int((v//32) % 32) +
+                                'expr ' * (v % 32) + opname)
                     self.add_unique_rule(rule, opname, token.attr, customize)
             elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
                 v = token.attr

uncompyle6/parsers/parse33.py

@@ -34,5 +34,16 @@ class Python33Parser(Python32Parser):
         jump_excepts ::= jump_except+
         """
 
+    def add_custom_rules(self, tokens, customize):
+        self.remove_rules("""
+        whileTruestmt ::= SETUP_LOOP l_stmts JUMP_ABSOLUTE JUMP_BACK COME_FROM_LOOP
+        whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP
+        whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP
+        whilestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK
+                          POP_BLOCK JUMP_ABSOLUTE COME_FROM_LOOP
+        """)
+        super(Python33Parser, self).add_custom_rules(tokens, customize)
+        return
+
 class Python33ParserSingle(Python33Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse34.py

@@ -24,13 +24,8 @@ class Python34Parser(Python33Parser):
         """
 
     def add_custom_rules(self, tokens, customize):
-        self.remove_rules("""
+        # self.remove_rules("""
-        whileTruestmt ::= SETUP_LOOP l_stmts JUMP_ABSOLUTE JUMP_BACK COME_FROM_LOOP
+        # """)
-        whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP
-        whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP
-        whilestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK
-                          POP_BLOCK JUMP_ABSOLUTE COME_FROM_LOOP
-        """)
         super(Python34Parser, self).add_custom_rules(tokens, customize)
         return
 

uncompyle6/semantics/pysource.py

@@ -981,8 +981,6 @@ class SourceWalker(GenericASTTraversal, object):
             self.write(iname, ' as ', sname)
         self.prune() # stop recursing
 
-    n_import_as_cont = n_import_as
-
     def n_importfrom(self, node):
         relative_path_index = 0
         if self.version >= 2.5 and node[relative_path_index].pattr > 0: