Add spark option to show grammer. Revise uncompyle options. Start to reorganize

tests more

bin/uncompyle6

@@ -32,8 +32,9 @@ Options:
   --help        show this message
 
 Debugging Options:
-  --showasm   -a  include byte-code                  (disables --verify)
-  --showast   -t  include AST (abstract syntax tree) (disables --verify)
+  --asm     -a  include byte-code         (disables --verify)
+  --grammar -g  show matching grammar
+  --treee   -t  include syntax tree       (disables --verify)
 
 Extensions of generated files:
   '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
@@ -51,7 +52,7 @@ from uncompyle6.main import main, status_msg
 
 def usage():
     print("""usage:
-   %s [--help] [--verify] [--showasm] [--showast] [-o <path>] FILE|DIR...
+   %s [--help] [--verify] [--asm] [--tree] [--grammar] [-o <path>] FILE|DIR...
 """  % program)
     sys.exit(1)
 
@@ -67,33 +68,37 @@ timestamp = True
 timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
 
 try:
-    opts, files = getopt.getopt(sys.argv[1:], 'hatdro:c:p:',
-                           ['help', 'verify', 'showast', 'showasm'])
+    opts, files = getopt.getopt(sys.argv[1:], 'hagtdro:c:p:',
+                                'help asm grammar recurse timestamp tree verify '
+                                'showgrammar'.split(' '))
 except getopt.GetoptError as e:
     print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
     sys.exit(-1)
 
+options = {}
 for opt, val in opts:
     if opt in ('-h', '--help'):
         print(__doc__)
         sys.exit(0)
     elif opt == '--verify':
-        do_verify = True
-    elif opt in ('--showasm', '-a'):
-        showasm = True
-        do_verify = False
-    elif opt in ('--showast', '-t'):
-        showast = True
-        do_verify = False
+        options['do_verify'] = True
+    elif opt in ('--asm', '-a'):
+        options['showasm'] = True
+        options['do_verify'] = False
+    elif opt in ('--tree', '-t'):
+        options['showast'] = True
+        options['do_verify'] = False
+    elif opt in ('--grammar', '-g'):
+        options['showgrammar'] = True
     elif opt == '-o':
         outfile = val
-    elif opt == '-d':
+    elif opt == ('--timestamp', '-d'):
         timestamp = False
     elif opt == '-c':
         codes.append(val)
     elif opt == '-p':
         numproc = int(val)
-    elif opt == '-r':
+    elif opt == ('--recurse', '-r'):
         recurse_dirs = True
     else:
         print(opt, file=sys.stderr)
@@ -137,8 +142,8 @@ if timestamp:
     print(time.strftime(timestampfmt))
 if numproc <= 1:
     try:
-        result = main(src_base, out_base, files, codes, outfile, showasm,
-                      showast, do_verify)
+        result = main(src_base, out_base, files, codes, outfile,
+                      **options)
         if len(files) > 1:
             mess = status_msg(do_verify, *result)
             print('# ' + mess)
@@ -171,7 +176,7 @@ else:
                 if f is None:
                     break
                 (t, o, f, v) = \
-                  main(src_base, out_base, [f], codes, outfile, showasm, showast, do_verify)
+                  main(src_base, out_base, [f], codes, outfile, **options)
                 tot_files += t
                 okay_files += o
                 failed_files += f

pytest/test_disasm.py

@@ -13,11 +13,11 @@ os.chdir(src_dir)
 
 @pytest.mark.parametrize(("test_tuple", "function_to_test"), [
     (
-        ('../test/bytecode_2.7/if.pyc', 'testdata/if-2.7.right',),
+        ('../test/bytecode_2.7/05_if.pyc', 'testdata/if-2.7.right',),
         disassemble_file
     ),
     (
-        ('../test/bytecode_2.7/ifelse.pyc', 'testdata/ifelse-2.7.right',),
+        ('../test/bytecode_2.7/05_ifelse.pyc', 'testdata/ifelse-2.7.right',),
         disassemble_file
     ),
 ])

pytest/testdata/if-2.7.right

@@ -1,10 +1,10 @@
 # Python 2.7
-# Embedded file name: ./source_3.4/branching/if.py
+# Embedded file name: simple_source/branching/05_if.py
 
-   1       0	LOAD_NAME         'a'
+   6       0	LOAD_NAME         'True'
            3	POP_JUMP_IF_FALSE '15'
 
-   2       6	LOAD_NAME         'c'
+   7       6	LOAD_NAME         'False'
            9	STORE_NAME        'b'
           12	JUMP_FORWARD      '15'
         15_0	COME_FROM         '12'

pytest/testdata/ifelse-2.7.right

@@ -1,14 +1,14 @@
 # Python 2.7
-# Embedded file name: ./source_3.4/branching/ifelse.py
+# Embedded file name: simple_source/branching/05_ifelse.py
 
-   1       0	LOAD_NAME         'a'
+   3       0	LOAD_NAME         'True'
            3	POP_JUMP_IF_FALSE '15'
 
-   2       6	LOAD_NAME         'c'
+   4       6	LOAD_CONST        1
            9	STORE_NAME        'b'
           12	JUMP_FORWARD      '21'
 
-   4      15	LOAD_NAME         'e'
+   6      15	LOAD_CONST        2
           18	STORE_NAME        'd'
         21_0	COME_FROM         '12'
           21	LOAD_CONST        ''

test/Makefile

@@ -46,9 +46,9 @@ check-bytecode-2.7:
 check-bytecode-3.2:
 	$(PYTHON) test_pythonlib.py --bytecode-3.2
 
-#: Check deparsing Python 3.2
+#: Check deparsing Python 3.4
 check-bytecode-3.4:
-	$(PYTHON) test_pythonlib.py --bytecode-3.2
+	$(PYTHON) test_pythonlib.py --bytecode-3.4
 
 #: short tests for bytecodes only for this version of Python
 check-native-short:

test/add-test.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+""" Trivial helper program to bytecompile and run an uncompile
+"""
+import os, sys, py_compile
+assert len(sys.argv) == 2
+path = sys.argv[1]
+short = os.path.basename(path)
+version = sys.version[0:3]
+cfile =  "bytecode_%s/%s" % (version, short) + 'c'
+print("byte-compiling %s to %s" % (path, cfile))
+py_compile.compile(path, cfile)
+os.system("../bin/uncompyle6 -a -t %s" % cfile)

test/simple_source/branching/05_if.py

@@ -0,0 +1,7 @@
+# Tests:
+
+#   ifstmt ::= testexpr _ifstmts_jump
+#   _ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
+
+if True:
+    b = False

test/simple_source/branching/05_ifelse.py

@@ -0,0 +1,9 @@
+# Tests:
+
+#   ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
+#   else_suite ::= suite_stmts
+
+if True:
+    b = 1
+else:
+    d = 2

test/simple_source/branching/if.py

@@ -1,4 +0,0 @@
-# Tests:
-# ifstmt ::= testexpr _ifstmts_jump
-if a:
-    b = c

test/simple_source/branching/ifelse.py

@@ -1,7 +0,0 @@
-
-# Tests
-# ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
-if a:
-    b = c
-else:
-    d = e

test/simple_source/call_arguments/01_positional.py

@@ -2,4 +2,4 @@
 #   expr ::= expr {expr}^n CALL_FUNCTION_n
 # which in the specifc case below is:
 #   expr ::= expr expr expr CALL_FUNCTION_2
-a(b, c)
+max(1, 2)

test/simple_source/call_arguments/10_keyword.py

@@ -0,0 +1,8 @@
+# Tests:
+#
+#   call_stmt ::= expr POP_TOP
+#   build_list ::= expr expr BUILD_LIST_2
+#   kwarg ::= LOAD_CONST expr
+#   call_function ::= expr expr kwarg kwarg CALL_FUNCTION_513
+
+sorted([1,2], reverse=True, key=None)

test/simple_source/call_arguments/keyword.py

@@ -1 +0,0 @@
-a = b(c=d, e=f)

test/simple_source/exception/20_try_except.py

@@ -1,3 +1,11 @@
+# Tests:
+#   forstmt ::= SETUP_LOOP expr _for designator for_block POP_BLOCK COME_FROM
+#   for_block ::= l_stmts_opt JUMP_BACK
+#   trystmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK try_middle COME_FROM
+#   try_middle ::= jmp_abs COME_FROM except_stmts END_FINALLY
+
+# Had a bug with the end of the except matching the end of the
+# for loop.
 for i in (1,2):
     try:
         x = 1

test/simple_source/exception/25_try_except.py

@@ -0,0 +1,21 @@
+# Tests:
+#
+#   tryfinallystmt ::= SETUP_FINALLY suite_stmts POP_BLOCK LOAD_CONST COME_FROM suite_stmts_opt END_FINALLY
+#   suite_stmts_opt ::= suite_stmts
+#   trystmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK try_middle COME_FROM
+#   try_middle ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY COME_FROM
+#   except_stmt ::= except_cond1 except_suite
+#   except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false POP_TOP POP_TOP POP_TOP
+#   trystmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK try_middle COME_FROM
+#   try_middle ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY COME_FROM
+#   except_cond1 ::= DUP_TOP expr COMPARE_OP jmp_false POP_TOP POP_TOP POP_TOP
+try:
+    try:
+        x = 1
+    except AssertionError:
+        x = 2
+
+except ImportError:
+    x = 3
+finally:
+    x = 4

test/simple_source/misc/assert.py

@@ -1,5 +0,0 @@
-# Tests:
-# assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
-# callfunction ::= ...
-
-assert isinstance(1, int)

test/simple_source/misc/assign_none.py

@@ -1 +0,0 @@
-vario = None

test/simple_source/operation_logic/01_boolean.py

@@ -0,0 +1,2 @@
+1 and 0
+1 or 0

test/simple_source/operation_logic/10_mixed_boolean.py

@@ -0,0 +1,12 @@
+b = True
+c = False
+d = True
+a = b and c or d
+a = (b or c) and d
+a = b or c or d
+a = b and c and d
+a = b or c and d
+a = b and (c or d)
+a = b and c or d
+a = (b or c and d) and b
+a = (b or c and d or a) and b

test/simple_source/operation_logic/or.py

@@ -1 +0,0 @@
-a = b or c

test/simple_source/simple_stmts/15_assert.py

@@ -0,0 +1,5 @@
+# Tests:
+#   assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
+#   call_function ::= expr expr expr CALL_FUNCTION_2
+
+assert isinstance(1, int)

test/simple_source/slice/02_slice.py

@@ -1,2 +1,3 @@
 ary = [1,2,3]
 ary[:2]
+ary[2:]

uncompyle6/main.py

@@ -9,7 +9,8 @@ from uncompyle6.scanner import get_scanner
 from uncompyle6.load import load_module
 
 # FIXME: remove duplicate code from deparse_code
-def uncompyle(version, co, out=None, showasm=False, showast=False):
+def uncompyle(version, co, out=None, showasm=False, showast=False,
+              showgrammar=False):
     """
     disassembles and deparses a given code block 'co'
     """
@@ -32,7 +33,8 @@ def uncompyle(version, co, out=None, showasm=False, showast=False):
         print(file=out)
 
     #  Build AST from disassembly.
-    walk = pysource.Walker(version, out, scanner, showast=showast)
+    walk = pysource.Walker(version, out, scanner,
+                           showast=showast, showgrammar=showgrammar)
     try:
         ast = walk.build_ast(tokens, customize)
     except pysource.ParserError as e :  # parser failed, dump disassembly

uncompyle6/parsers/parse3.py

@@ -19,13 +19,13 @@ from __future__ import print_function
 
 from uncompyle6.parser import PythonParser, nop_func
 from uncompyle6.parsers.astnode import AST
-from uncompyle6.parsers.spark import GenericASTBuilder
+from uncompyle6.parsers.spark import GenericASTBuilder, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
 class Python3Parser(PythonParser):
 
-    def __init__(self):
+    def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         self.added_rules = set()
-        GenericASTBuilder.__init__(self, AST, 'stmts')
+        GenericASTBuilder.__init__(self, AST, 'stmts', debug=debug_parser)
         self.new_rules = set()
 
     def add_unique_rule(self, rule, opname, count, customize):

uncompyle6/parsers/spark.py

@@ -45,7 +45,9 @@ class _State:
         self.stateno = stateno
 
 # DEFAULT_DEBUG = {'rules': True, 'transition': True, 'reduce' : True}
+# DEFAULT_DEBUG = {'rules': False, 'transition': False, 'reduce' : True}
 DEFAULT_DEBUG = {'rules': False, 'transition': False, 'reduce': False}
+
 class GenericParser:
     '''
     An Earley parser, as per J. Earley, "An Efficient Context-Free
@@ -640,7 +642,7 @@ class GenericParser:
 #
 
 class GenericASTBuilder(GenericParser):
-    def __init__(self, AST, start):
+    def __init__(self, AST, start, debug=DEFAULT_DEBUG):
         GenericParser.__init__(self, start)
         self.AST = AST
 

uncompyle6/semantics/pysource.py

@@ -68,7 +68,7 @@ import inspect, sys, re
 from uncompyle6 import PYTHON3
 from uncompyle6.parser import get_python_parser
 from uncompyle6.parsers.astnode import AST
-from uncompyle6.parsers.spark import GenericASTTraversal
+from uncompyle6.parsers.spark import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from uncompyle6.scanner import Code, get_scanner
 from uncompyle6.scanners.tok import Token, NoneToken
 import uncompyle6.parser as python_parser
@@ -470,7 +470,7 @@ def find_none(node):
 class Walker(GenericASTTraversal, object):
     stacked_params = ('f', 'indent', 'isLambda', '_globals')
 
-    def __init__(self, version, out, scanner, showast=False):
+    def __init__(self, version, out, scanner, showast=False, showgrammar=False):
         GenericASTTraversal.__init__(self, ast=None)
         self.scanner = scanner
         params = {
@@ -1532,7 +1532,8 @@ class Walker(GenericASTTraversal, object):
 
         return ast
 
-def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False):
+def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False,
+                 showgrammar=False):
     """
     disassembles and deparses a given code block 'co'
     """
@@ -1547,6 +1548,10 @@ def deparse_code(version, co, out=sys.stdout, showasm=False, showast=False):
         for t in tokens:
             print(t)
 
+    parser_debug = dict(PARSER_DEFAULT_DEBUG)
+    parser_debug['reduce'] = showgrammar
+
+
     #  Build AST from disassembly.
     walk = Walker(version, out, scanner, showast=showast)