Get ready for release 3.1.0

remove parens around decorators by adjusting precidence
Partial handling of quotes within 3.6 format strings

.travis.yml

@@ -4,7 +4,7 @@ sudo: false
 
 python:
   - '3.5'
-  - '2.7.12'
+  - '2.7'
   - '2.6'
   - '3.3'
   - '3.4'

NEWS

@@ -1,3 +1,11 @@
+uncompyle6 3.1.0 2018-03-21 Equinox
+
+- Add code_deparse_with_offset() fragment function.
+- Correct paramenter call fragment deparse_code()
+- Lots of 3.6, 3.x, and 2.7 bug fixes
+  About 5% of 3.6 fail parsing now. But
+  semantics still needs much to be desired.
+
 uncompyle6 3.0.1 2018-02-17
 
 - All Python 2.6.9 standard library files weakly verify

__pkginfo__.py

@@ -35,8 +35,6 @@ classifiers =  ['Development Status :: 5 - Production/Stable',
                 'Programming Language :: Python :: 2.5',
                 'Programming Language :: Python :: 2.6',
                 'Programming Language :: Python :: 2.7',
-                'Programming Language :: Python :: 3.1',
-                'Programming Language :: Python :: 3.2',
                 'Programming Language :: Python :: 3.3',
                 'Programming Language :: Python :: 3.4',
                 'Programming Language :: Python :: 3.5',

test/run-and-email.sh

@@ -1,13 +1,65 @@
-#!/usr/bin/bash
+#!/bin/bash
+
+function displaytime {
+    printf "ran in "
+    local T=$1
+    local D=$((T/60/60/24))
+    local H=$((T/60/60%24))
+    local M=$((T/60%60))
+    local S=$((T%60))
+    (( $D > 0 )) && printf '%d days ' $D
+    (( $H > 0 )) && printf '%d hours ' $H
+    (( $M > 0 )) && printf '%d minutes ' $M
+    (( $D > 0 || $H > 0 || $M > 0 )) && printf 'and '
+    printf '%d seconds\n' $S
+}
+
+PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.4.8 2.6.9"}
+# PYVERSION=${PYVERSION:-"3.5.5"}
+
+USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
-for VERSION in 2.7.14 2.6.9 ; do
-    LOGFILE=/tmp/pyenlib-$VERSION-$$.log
-    python ./test_pyenvlib.py --max 800 --weak-verify --$VERSION  >$LOGFILE 2>&1
-    rc=$?
-    if ((rc == 0)); then
-	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+MAX_TESTS=${MAX_TESTS:-800}
+typeset -i RUN_STARTTIME=$(date +%s)
+
+for VERSION in $PYVERSION ; do
+    typeset -i rc=0
+    LOGFILE=/tmp/pyenvlib-$VERSION-$$.log
+
+    if [[ $VERSION == '3.5.5' ]] ; then
+	MAX_TESTS=224
     else
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+	MAX_TESTS=800
     fi
+
+    if ! pyenv local $VERSION ; then
+	rc=1
+    else
+      echo Python Version $(pyenv local) > $LOGFILE
+      echo "" >> $LOGFILE
+      typeset -i ALL_FILES_STARTTIME=$(date +%s)
+      python ./test_pyenvlib.py --max ${MAX_TESTS} --weak-verify --$VERSION  >>$LOGFILE 2>&1
+      rc=$?
+
+      echo Python Version $(pyenv local) >> $LOGFILE
+      echo "" >>LOGFILE
+
+      typeset -i ALL_FILES_ENDTIME=$(date +%s)
+      (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
+      displaytime $time_diff >> $LOGFILE
+    fi
+
+    SUBJECT_PREFIX="pyenv weak verify (max $MAX_TESTS) for"
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
+    else
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${EMAIL}
+    fi
+    rm .python-version
 done
+
+typeset -i RUN_ENDTIME=$(date +%s)
+(( time_diff =  RUN_ENDTIME - RUN_STARTTIME))
+elapsed_time=$(displaytime $time_diff)
+echo "Run complete $elapsed_time for versions $PYVERSION" | mail -s "pyenv weak verify in $elapsed_time" ${EMAIL}

test/simple_source/bug27+/02_assert.py

@@ -0,0 +1,17 @@
+# From 2.7 test_argparse.py
+# Bug was turnning assert into an "or raise" statement
+def __call__(arg, dest):
+    try:
+        assert arg == 'spam', 'dest: %s' % dest
+    except:
+        raise
+
+__call__('spam', __file__)
+
+# From python 2.7.14 lib2to3/refactor.py
+# Bug was mangling assert turning if into "or"
+def refactor_doctest(clipped, new):
+    assert clipped, clipped
+    if not new:
+        new += u"\n"
+    return

test/simple_source/bug33/05_nonlocal.py

@@ -0,0 +1,12 @@
+# From Python 3.6 functools.py
+# Bug was in detecting "nonlocal" access
+def not_bug():
+    cache_token = 5
+
+    def register():
+        nonlocal cache_token
+        return cache_token == 5
+
+    return register()
+
+assert not_bug()

test/simple_source/bug34/05_while1_if_continue.py

@@ -0,0 +1,37 @@
+# Bug in Python 3.4 text_file.py
+# Bug is handling:  while true ... if ... continue
+def readline(b):
+    a = 1
+    while True:
+        if b:
+            if b[0]:
+                a = 2
+                b = None
+                continue
+        b = None
+        a = 5
+
+        return a
+
+assert readline(None) == 1
+assert readline([2]) == 2
+
+def readline2(self):
+    while True:
+        line = 5
+        if self[0]:
+            if self:
+                self[0] = 1
+                continue
+
+        return line + self[0]
+
+# From 3.4.4 connection.py
+def PipeClient(address):
+    while 1:
+        try:
+            address += 1
+        except OSError as e:
+            raise e
+    else:
+        raise

test/simple_source/bug36/01_fstring.py

@@ -1,5 +1,18 @@
+# Self-checking 3.6+ string interpolation tests
+
 var1 = 'x'
 var2 = 'y'
-print(f'interpolate {var1} strings {var2!r} {var2!s} py36')
-print(f'{abc}0')
-print(f'{abc}{abc!s}')
+abc  = 'def'
+assert (f'interpolate {var1} strings {var2!r} {var2!s} py36' ==
+        "interpolate x strings 'y' y py36")
+assert 'def0' == f'{abc}0'
+assert 'defdef' == f'{abc}{abc!s}'
+
+# From 3.6 functools.py
+# Bug was handling format operator strings.
+
+k, v = "1", ["2"]
+x = f"{k}={v!r}"
+y = f"functools.{x}({', '.join(v)})"
+assert x == "1=['2']"
+assert y == "functools.1=['2'](2)"

test/simple_source/bug36/02_kwargs.py

@@ -1,5 +1,13 @@
 # From 3.6 _markupbase _parse_doctype_subset()
-def bug(self, j):
+def bug(self, j, a, b):
     self.parse_comment(j, report=0)
     self.parse_comment(j, report=1, foo=2)
     self.parse_comment(a, b, report=3)
+
+# From 3.6 fnmatch.py
+# Bug was precidence parenthesis around decorator
+
+import functools
+@functools.lru_cache(maxsize=256, typed=True)
+def _compile_pattern(pat):
+    pass

test/simple_source/bug36/03_try_return_except.py

@@ -0,0 +1,30 @@
+# From Python 3.6 bdb.py
+# Bug was handling try's with returns
+# END_FINALLY in 3.6 starts disasppearing
+
+def effective(possibles):
+    for b in possibles:
+        try:
+            return 1
+        except:
+            return 2
+    return 3
+
+assert effective([5]) == 1
+assert effective([]) == 3
+
+def effective2(possibles):
+    b = 0
+    for b in possibles:
+        try:
+            if b >= 5:
+                b = 5
+            else:
+                return 2
+        except:
+            return 3
+    return b
+
+assert effective2([5]) == 5
+assert effective2([]) == 0
+assert effective2(['a']) == 3

test/simple_source/bug36/04_class_kwargs.py

@@ -0,0 +1,17 @@
+# From 3.6 test_abc.py
+# Bug was Reciever() class definition
+import abc
+import unittest
+class TestABCWithInitSubclass(unittest.TestCase):
+    def test_works_with_init_subclass(self):
+        class ReceivesClassKwargs:
+            def __init_subclass__(cls, **kwargs):
+                super().__init_subclass__()
+        class Receiver(ReceivesClassKwargs, abc.ABC, x=1, y=2, z=3):
+            pass
+
+def test_abstractmethod_integration(self):
+    for abstractthing in [abc.abstractmethod]:
+        class C(metaclass=abc.ABCMeta):
+            @abstractthing
+            def foo(self): pass  # abstract

test/simple_source/bug36/04_try_finally.py

@@ -18,3 +18,16 @@ def getvalue1(self):
     finally:
         pass
     return 2
+
+# From Python 3.6 asynchat.py
+# Bug is handling as why in the face of a return.
+# uncompyle6 shows removal of "why" after the return.
+def handle_read(self):
+    try:
+        data = 5
+    except ZeroDivisionError:
+        return
+    except OSError as why:
+        return why
+
+    return data

test/simple_source/bug36/05_36lambda.py

@@ -0,0 +1,20 @@
+# From Python 3.6 hmac.py
+# needed to change mklambda rule
+def __init__(self, msg = None, digestmod = None):
+    self.digest_cons = lambda d='': digestmod.new(d)
+
+# From Python 3.6 functools.py
+# Bug was handling lambda for MAKE_FUNCTION_8 (closure)
+# vs to MAKE_FUNCTION_9 (pos_args + closure)
+def bug():
+    def register(cls, func=None):
+        return lambda f: register(cls, f)
+
+# From Python 3.6 configparser.py
+def items(self, d, section=5, raw=False, vars=None):
+    if vars:
+        for key, value in vars.items():
+            d[self.optionxform(key)] = value
+    d = lambda option: self._interpolation.before_get(self,
+        section, option, d[option], d)
+    return

test/simple_source/bug36/05_if_and_comp.py

@@ -0,0 +1,12 @@
+# From 3.6 base64.py
+# Bug was handling "and" condition in the presense of POP_JUMP_IF_FALSE
+# locations
+def _85encode(foldnuls, words):
+    return ['z' if foldnuls and word
+                else  'y'
+                for word in words]
+
+# From Python 3.6 enum.py
+
+def __new__(metacls, cls, bases, classdict):
+    {k: classdict[k] for k in classdict._member_names}

test/simple_source/bug36/05_while_and_if.py

@@ -0,0 +1,13 @@
+# From Python 3.6 getopt.py
+# Bug showing that "while" can have several "COME_FROMS" before loop end
+# NOTE: uncompyle6 still gets the "if"s wrong.
+def getopt(args):
+    while args and args[0] and args[0] != '-':
+        if args[0] == '--':
+            break
+        if args[0]:
+            opts = 5
+        else:
+            opts = 6
+
+    return opts

test/stdlib/run-and-email.sh

@@ -1,13 +1,20 @@
-#!/usr/bin/bash
+#!/bin/bash
+USER=${USER:-rocky}
 EMAIL=${EMAIL:-rb@dustyfeet.com}
+SUBJECT_PREFIX="stdlib unit testing for"
 for VERSION in 2.7.14 2.6.9 ; do
+    typeset -i rc=0
     LOGFILE=/tmp/runtests-$VERSION-$$.log
-    ./runtests.sh  >$LOGFILE 2>&1
-    rc=$?
-    if ((rc == 0)); then
-	tail -v $LOGFILE | mail -s \""$VERSION ok"\" rocky@localhost
+    if ! pyenv local $VERSION ; then
+	rc=1
     else
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rocky@localhost
-	tail -v $LOGFILE | mail -s \""$VERSION not ok"\" rb@dustyfeet.com
+      /bin/bash ./runtests.sh  >$LOGFILE 2>&1
+      rc=$?
+    fi
+    if ((rc == 0)); then
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION ok" ${USER}@localhost
+    else
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" ${USER}@localhost
+	tail -v $LOGFILE | mail -s "$SUBJECT_PREFIX $VERSION not ok" $EMAIL
     fi
 done

test/stdlib/runtests.sh

@@ -48,6 +48,7 @@ case $PYVERSION in
 	;;
     2.6)
 	SKIP_TESTS=(
+	    [test_compile.py]=1  # Intermittent - sometimes works and sometimes doesn't
 	    [test_grp.py]=1      # Long test - might work Control flow?
 	    [test_opcodes.py]=1
 	    [test_pwd.py]=1 # Long test - might work? Control flow?
@@ -96,9 +97,23 @@ case $PYVERSION in
 	    [test_zipfile64.py]=1  # Runs ok but takes 204 seconds
         )
 	;;
+    3.5)
+	SKIP_TESTS=(
+	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	)
+	;;
+    3.6)
+	SKIP_TESTS=(
+	    [test_contains.py]=1    # Code "while False: yield None" is optimized away in compilation
+	    [test_decorators.py]=1  # Control flow wrt "if elif"
+	    [test_pow.py]=1         # Control flow wrt "continue"
+	)
+	;;
     *)
-	SKIP_TESTS=( [test_aepack.py]=1 [audiotests.py]=1
+	SKIP_TESTS=( [test_aepack.py]=1
+		     [audiotests.py]=1
 		     [test_dis.py]=1   # We change line numbers - duh!
+		     [test_generators.py]=1  # I think string formatting of docstrings gets in the way. Not sure
 		   )
 	;;
 esac
@@ -178,7 +193,7 @@ typeset -i ALL_FILES_ENDTIME=$(date +%s)
 
 (( time_diff =  ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))
 
-printf "Ran $i tests in "
+printf "Ran $i unit-test files in "
 displaytime $time_diff
 
 exit $allerrs

uncompyle6/main.py

@@ -38,7 +38,11 @@ def _get_outstream(outfile):
         os.makedirs(dir)
     except OSError:
         pass
-    return open(outfile, 'w')
+    if PYTHON_VERSION < 3.0:
+        mode = 'wb'
+    else:
+        mode = 'w'
+    return open(outfile, mode)
 
 def decompile(
         bytecode_version, co, out=None, showasm=None, showast=False,

uncompyle6/parser.py

@@ -431,9 +431,6 @@ class PythonParser(GenericASTBuilder):
 
         for_block ::= l_stmts_opt _come_froms JUMP_BACK
 
-        for       ::= SETUP_LOOP expr for_iter store
-                      for_block POP_BLOCK _come_froms
-
         forelsestmt ::= SETUP_LOOP expr for_iter store
                         for_block POP_BLOCK else_suite _come_froms
 

uncompyle6/parsers/parse2.py

@@ -1,8 +1,21 @@
 #  Copyright (c) 2015-2017 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
+#
 #  Copyright (c) 1999 John Aycock
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
-A spark grammar for Python 2.x.
+Base grammar for Python 2.x.
 
 However instead of terminal symbols being the usual ASCII text,
 e.g. 5, myvariable, "for", etc.  they are CPython Bytecode tokens,
@@ -82,6 +95,9 @@ class Python2Parser(PythonParser):
         raise_stmt2 ::= expr expr RAISE_VARARGS_2
         raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
 
+        for         ::= SETUP_LOOP expr for_iter store
+                        for_block POP_BLOCK _come_froms
+
         del_stmt ::= expr DELETE_SLICE+0
         del_stmt ::= expr expr DELETE_SLICE+1
         del_stmt ::= expr expr DELETE_SLICE+2
@@ -175,6 +191,7 @@ class Python2Parser(PythonParser):
     def p_expr2(self, args):
         """
         expr ::= LOAD_LOCALS
+        expr ::= LOAD_ASSERT
         expr ::= slice0
         expr ::= slice1
         expr ::= slice2
@@ -505,6 +522,7 @@ class Python2Parser(PythonParser):
 
         self.check_reduce['aug_assign1'] = 'AST'
         self.check_reduce['aug_assign2'] = 'AST'
+        self.check_reduce['or'] = 'AST'
         # self.check_reduce['_stmts'] = 'AST'
 
         # Dead code testing...
@@ -520,8 +538,11 @@ class Python2Parser(PythonParser):
         # if lhs == 'while1elsestmt':
         #     from trepan.api import debug; debug()
 
-        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] == 'and':
+        if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
             return True
+        if rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
+            expr2 = ast[2]
+            return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'
         return False
 
 class Python2ParserSingle(Python2Parser, PythonParserSingle):

uncompyle6/parsers/parse27.py

@@ -175,6 +175,7 @@ class Python27Parser(Python2Parser):
         """)
         super(Python27Parser, self).customize_grammar_rules(tokens, customize)
         self.check_reduce['and'] = 'AST'
+        self.check_reduce['raise_stmt1'] = 'AST'
         # self.check_reduce['conditional_true'] = 'AST'
         return
 
@@ -191,6 +192,8 @@ class Python27Parser(Python2Parser):
             jmp_target = jmp_false.offset + jmp_false.attr + 3
             return not (jmp_target == tokens[last].offset or
                         tokens[last].pattr == jmp_false.pattr)
+        elif rule[0] == ('raise_stmt1'):
+            return ast[0] == 'expr' and ast[0][0] == 'or'
         # elif rule[0] == ('conditional_true'):
         #     # FIXME: the below is a hack: we check expr for
         #     # nodes that could have possibly been a been a Boolean.

uncompyle6/parsers/parse3.py

@@ -240,10 +240,10 @@ class Python3Parser(PythonParser):
         except_suite ::= returns
 
         except_cond1 ::= DUP_TOP expr COMPARE_OP
-                jmp_false POP_TOP POP_TOP POP_TOP
+                         jmp_false POP_TOP POP_TOP POP_TOP
 
         except_cond2 ::= DUP_TOP expr COMPARE_OP
-                jmp_false POP_TOP store POP_TOP
+                         jmp_false POP_TOP store POP_TOP
 
         except  ::=  POP_TOP POP_TOP POP_TOP c_stmts_opt POP_EXCEPT _jump
         except  ::=  POP_TOP POP_TOP POP_TOP returns
@@ -886,6 +886,20 @@ class Python3Parser(PythonParser):
                     # before.
                     args_pos, args_kw, annotate_args, closure  = token.attr
                     stack_count = args_pos + args_kw + annotate_args
+                    if closure:
+                        if args_pos:
+                            rule = ('mklambda ::= %s%s%s%s' %
+                                        ('expr ' * stack_count,
+                                         'load_closure ' * closure,
+                                         'BUILD_TUPLE_1 LOAD_LAMBDA LOAD_CONST ',
+                                        opname))
+                        else:
+                            rule = ('mklambda ::= %s%s%s' %
+                                        ('load_closure ' * closure,
+                                         'LOAD_LAMBDA LOAD_CONST ',
+                                        opname))
+                        self.add_unique_rule(rule, opname, token.attr, customize)
+
                     rule = ('mkfunc ::= %s%s%s%s' %
                                 ('expr ' * stack_count,
                                  'load_closure ' * closure,
@@ -1076,7 +1090,7 @@ class Python3Parser(PythonParser):
             if tokens[last] in ('JUMP_BACK', 'CONTINUE'):
                 # These indicate inside a loop, but token[last]
                 # should not be in a loop.
-                # FIXME: Not quite righte: refine by using target
+                # FIXME: Not quite right: refine by using target
                 return True
 
             # if SETUP_LOOP target spans the else part, then this is
@@ -1086,7 +1100,7 @@ class Python3Parser(PythonParser):
                 last += 1
             if last == n:
                 return False
-            return tokens[first].attr >= tokens[last].offset
+            return tokens[first].attr > tokens[last].offset
         elif lhs == 'while1stmt':
 
             # If there is a fall through to the COME_FROM_LOOP. then this is

uncompyle6/parsers/parse34.py

@@ -34,16 +34,23 @@ class Python34Parser(Python33Parser):
         # Seems to be needed starting 3.4.4 or so
         while1stmt    ::= SETUP_LOOP l_stmts
                           COME_FROM JUMP_BACK POP_BLOCK COME_FROM_LOOP
+        while1stmt    ::= SETUP_LOOP l_stmts
+                          POP_BLOCK COME_FROM_LOOP
 
         # FIXME the below masks a bug in not detecting COME_FROM_LOOP
         # grammar rules with COME_FROM -> COME_FROM_LOOP already exist
         whileelsestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               else_suitel COME_FROM
 
+        while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK else_suitel
+                              COME_FROM_LOOP
+
         # Python 3.4+ optimizes the trailing two JUMPS away
 
         # Is this 3.4 only?
         yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
+
+        _ifstmts_jump ::= c_stmts_opt JUMP_ABSOLUTE JUMP_FORWARD COME_FROM
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/parse36.py

@@ -33,9 +33,12 @@ class Python36Parser(Python35Parser):
         """
         sstmt ::= sstmt RETURN_LAST
 
-        # 3.6 redoes how return_closure works
+        # 3.6 redoes how return_closure works. FIXME: Isolate to LOAD_CLOSURE
         return_closure   ::= LOAD_CLOSURE DUP_TOP STORE_NAME RETURN_VALUE RETURN_LAST
 
+        # Is there something general going on here? FIXME: Isolate to LOAD_DICTCOMP
+        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_FUNCTION_8 expr GET_ITER CALL_FUNCTION_1
+
         stmt               ::= conditional_lambda
         conditional_lambda ::= expr jmp_false expr return_if_lambda
                                return_stmt_lambda LAMBDA_MARKER
@@ -47,11 +50,14 @@ class Python36Parser(Python35Parser):
         come_from_loops ::= COME_FROM_LOOP*
 
         whilestmt       ::= SETUP_LOOP testexpr l_stmts_opt
-                            JUMP_BACK COME_FROM POP_BLOCK COME_FROM_LOOP
+                            JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
 
         # This might be valid in < 3.6
         and  ::= expr jmp_false expr
 
+        jf_cf       ::= JUMP_FORWARD COME_FROM
+        conditional ::= expr jmp_false expr jf_cf expr COME_FROM
+
         # Adds a COME_FROM_ASYNC_WITH over 3.5
         # FIXME: remove corresponding rule for 3.5?
 
@@ -69,16 +75,26 @@ class Python36Parser(Python35Parser):
         stmt             ::= try_except36
         try_except36     ::= SETUP_EXCEPT returns except_handler36
                              opt_come_from_except
+        try_except36     ::= SETUP_EXCEPT suite_stmts
+
+        # 3.6 omits END_FINALLY sometimes
+        except_handler36 ::= COME_FROM_EXCEPT except_stmts
+        except_handler   ::= jmp_abs COME_FROM_EXCEPT except_stmts
 
         stmt             ::= tryfinally36
         tryfinally36     ::= SETUP_FINALLY returns
                              COME_FROM_FINALLY suite_stmts
         tryfinally36     ::= SETUP_FINALLY returns
                              COME_FROM_FINALLY suite_stmts_opt END_FINALLY
+        except_suite_finalize ::= SETUP_FINALLY returns
+                                  COME_FROM_FINALLY suite_stmts_opt END_FINALLY _jump
         """
 
     def customize_grammar_rules(self, tokens, customize):
         super(Python36Parser, self).customize_grammar_rules(tokens, customize)
+        self.remove_rules("""
+           dict_comp ::= load_closure LOAD_DICTCOMP LOAD_CONST MAKE_CLOSURE_0 expr GET_ITER CALL_FUNCTION_1
+        """)
         self.check_reduce['call_kw'] = 'AST'
 
         for i, token in enumerate(tokens):

uncompyle6/scanners/scanner2.py

@@ -37,6 +37,7 @@ from __future__ import print_function
 
 from collections import namedtuple
 from array import array
+from copy import copy
 
 from xdis.code import iscode
 from xdis.bytecode import (
@@ -54,6 +55,7 @@ class Scanner2(Scanner):
         # This is the 2.5+ default
         # For <2.5 it is <generator expression>
         self.genexpr_name = '<genexpr>'
+        self.load_asserts = set([])
 
     @staticmethod
     def unmangle_name(name, classname):
@@ -139,6 +141,9 @@ class Scanner2(Scanner):
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
         for i in self.op_range(0, codelen):
+
+            self.offset2inst_index[inst.offset] = i
+
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -159,7 +164,9 @@ class Scanner2(Scanner):
 
         # Get jump targets
         # Format: {target offset: [jump offsets]}
+        load_asserts_save = copy(self.load_asserts)
         jump_targets = self.find_jump_targets(show_asm)
+        self.load_asserts = load_asserts_save
         # print("XXX2", jump_targets)
 
         last_stmt = self.next_stmt[0]

uncompyle6/semantics/consts.py

@@ -317,6 +317,10 @@ MAP = {
 # See https://docs.python.org/2/reference/expressions.html
 # or https://docs.python.org/3/reference/expressions.html
 # for a list.
+
+# Things at the top of this list below with low-value precidence will
+# tend to have parenthesis around them. Things at the bottom
+# of the list will tend not to have parenthesis around them.
 PRECEDENCE = {
     'list':                 0,
     'dict':                 0,
@@ -377,8 +381,9 @@ PRECEDENCE = {
     'ret_cond_not':         28,
 
     '_mklambda':            30,
-    'yield':                101,
-    'yield_from':           101
+    'call_kw':             100,  # 100 seems to to be module/function precidence
+    'yield':               101,
+    'yield_from':          101
 }
 
 ASSIGN_TUPLE_PARAM = lambda param_name: \

uncompyle6/semantics/customize.py

@@ -357,8 +357,10 @@ def customize_for_version(self, is_pypy, version):
                     'tryfinally36':  ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
                                        (1, 'returns'), 3 ),
                     'fstring_expr':   ( "{%c%{conversion}}", 0),
-                    'fstring_single': ( "f'{%c%{conversion}}'", 0),
-                    'fstring_multi':  ( "f'%c'", 0),
+                    # FIXME: the below assumes the format strings
+                    # don't have ''' in them. Fix this properly
+                    'fstring_single': ( "f'''{%c%{conversion}}'''", 0),
+                    'fstring_multi':  ( "f'''%c'''", 0),
                     'func_args36':    ( "%c(**", 0),
                     'try_except36':   ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
                     'unpack_list':    ( '*%c', (0, 'list') ),
@@ -603,6 +605,14 @@ def customize_for_version(self, is_pypy, version):
 
                 FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
 
+                def n_formatted_value(node):
+                    if node[0] == 'LOAD_CONST':
+                        self.write(node[0].attr)
+                        self.prune()
+                    else:
+                        self.default(node)
+                self.n_formatted_value = n_formatted_value
+
                 def f_conversion(node):
                     node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '')
 

uncompyle6/semantics/fragments.py

@@ -65,9 +65,10 @@ The node position 0 will be associated with "import".
 
 from __future__ import print_function
 
-import re, sys
+import re
 
 from xdis.code import iscode
+from xdis.magics import sysinfo2float
 from uncompyle6.semantics import pysource
 from uncompyle6 import parser
 from uncompyle6.scanner import Token, Code, get_scanner
@@ -1734,8 +1735,12 @@ def deparse_code(version, co, out=StringIO(), showasm=False, showast=False,
         'ast': showast,
         'grammar': showgrammar
     }
-    return code_deparse(co, out, version, debug_opts, code_objects, compile_mode,
-                        is_pypy, walker)
+    return code_deparse(co, out,
+                        version=version,
+                        debug_opts=debug_opts,
+                        code_objects=code_objects,
+                        compile_mode=compile_mode,
+                        is_pypy=is_pypy, walker=walker)
 
 def code_deparse(co, out=StringIO(), version=None, is_pypy=None,
                  debug_opts=DEFAULT_DEBUG_OPTS,
@@ -1763,7 +1768,7 @@ def code_deparse(co, out=StringIO(), version=None, is_pypy=None,
     assert iscode(co)
 
     if version is None:
-        version = float(sys.version[0:3])
+        version = sysinfo2float()
     if is_pypy is None:
         is_pypy = IS_PYPY
 
@@ -1797,7 +1802,11 @@ def code_deparse(co, out=StringIO(), version=None, is_pypy=None,
 
     # convert leading '__doc__ = "..." into doc string
     assert deparsed.ast == 'stmts'
-    deparsed.mod_globs = pysource.find_globals(deparsed.ast, set())
+    (deparsed.mod_globs,
+     nonlocals) = (pysource
+                   .find_globals_and_nonlocals(deparsed.ast,
+                                               set(), set(),
+                                               co, version))
 
     # Just when you think we've forgotten about what we
     # were supposed to to: Generate source from AST!
@@ -1836,15 +1845,22 @@ def find_gt(a, x):
         return a[i]
     raise ValueError
 
-def deparse_code_around_offset(name, offset, version, co, out=StringIO(),
-                               showasm=False, showast=False,
-                               showgrammar=False, is_pypy=False):
+def code_deparse_around_offset(name, offset, co, out=StringIO(),
+                               version=None, is_pypy=None,
+                               debug_opts=DEFAULT_DEBUG_OPTS):
     """
     Like deparse_code(), but given  a function/module name and
     offset, finds the node closest to offset. If offset is not an instruction boundary,
     we raise an IndexError.
     """
-    deparsed = deparse_code(version, co, out, showasm, showast, showgrammar, is_pypy)
+    assert iscode(co)
+
+    if version is None:
+        version = sysinfo2float()
+    if is_pypy is None:
+        is_pypy = IS_PYPY
+
+    deparsed = code_deparse(co, out, version, is_pypy, debug_opts)
     if (name, offset) in deparsed.offsets.keys():
         # This is the easy case
         return deparsed
@@ -1857,6 +1873,17 @@ def deparse_code_around_offset(name, offset, version, co, out=StringIO(),
     deparsed.offsets[name, offset] = deparsed.offsets[name, found_offset]
     return deparsed
 
+# Deprecated. Here still for compatability
+def deparse_code_around_offset(name, offset, version, co, out=StringIO(),
+                               showasm=False, showast=False,
+                               showgrammar=False, is_pypy=False):
+    debug_opts = {
+        'asm': showasm,
+        'ast': showast,
+        'grammar': showgrammar
+    }
+    return code_deparse(name, offset, co, out, version, is_pypy,
+                        debug_opts)
 
 def op_at_code_loc(code, loc, opc):
     """Return the instruction name at code[loc] using
@@ -1926,12 +1953,7 @@ def deparsed_find(tup, deparsed, code):
 #         return
 
 #     def deparse_test_around(offset, name, co, is_pypy=IS_PYPY):
-#         sys_version = sys.version_info.major + (sys.version_info.minor / 10.0)
-#         deparsed = deparse_code_around_offset(name, offset, sys_version, co,
-#                                               showasm=False,
-#                                               showast=False,
-#                                               showgrammar=False,
-#                                               is_pypy=IS_PYPY)
+#         deparsed = code_deparse_around_offset(name, offset, co)
 #         print("deparsed source")
 #         print(deparsed.text, "\n")
 #         print('------------------------')

uncompyle6/semantics/helper.py

@@ -13,6 +13,9 @@ else:
 read_write_global_ops = frozenset(('STORE_GLOBAL', 'DELETE_GLOBAL', 'LOAD_GLOBAL'))
 read_global_ops       = frozenset(('STORE_GLOBAL', 'DELETE_GLOBAL'))
 
+# NOTE: we also need to check that the variable name is a free variable, not a cell variable.
+nonglobal_ops         = frozenset(('STORE_DEREF',  'DELETE_DEREF'))
+
 # FIXME: this and find_globals could be paramaterized with one of the
 # above global ops
 def find_all_globals(node, globs):
@@ -24,15 +27,22 @@ def find_all_globals(node, globs):
             globs.add(n.pattr)
     return globs
 
-def find_globals(node, globs):
+def find_globals_and_nonlocals(node, globs, nonlocals, code, version):
     """search a node of parse tree to find variable names that need a
-    'global' added."""
+    either 'global' or 'nonlocal' statements added."""
     for n in node:
         if isinstance(n, AST):
-            globs = find_globals(n, globs)
+            globs, nonlocals = find_globals_and_nonlocals(n, globs, nonlocals,
+                                                          code, version)
         elif n.kind in read_global_ops:
             globs.add(n.pattr)
-    return globs
+        elif (version >= 3.0
+              and n.kind in nonglobal_ops
+              and n.pattr in code.co_freevars
+              and n.pattr != code.co_name
+              and code.co_name != '<lambda>'):
+            nonlocals.add(n.pattr)
+    return globs, nonlocals
 
 # def find_globals(node, globs, global_ops=mkfunc_globals):
 #     """Find globals in this statement."""

uncompyle6/semantics/make_function.py

@@ -23,7 +23,7 @@ from uncompyle6 import PYTHON3
 from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.parser import ParserError as ParserError2
 from uncompyle6.semantics.helper import (
-    print_docstring, find_all_globals, find_globals, find_none
+    print_docstring, find_all_globals, find_globals_and_nonlocals, find_none
     )
 
 if PYTHON3:
@@ -269,8 +269,12 @@ def make_function3_annotate(self, node, is_lambda, nested=1,
     assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
-    for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())):
+    globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),
+                                                    code, self.version)
+    for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
+    for nl in sorted(nonlocals):
+        self.println(self.indent, 'nonlocal ', nl)
     self.mod_globs -= all_globals
     has_none = 'None' in code.co_names
     rn = has_none and not find_none(ast)
@@ -422,7 +426,17 @@ def make_function2(self, node, is_lambda, nested=1, codeNode=None):
         assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
-    for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())):
+
+    globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),
+                                                    code, self.version)
+
+    # Python 2 doesn't support the "nonlocal" statement
+    try:
+        assert self.version >= 3.0 or not nonlocals
+    except:
+        from trepan.api import debug; debug()
+
+    for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
     self.mod_globs -= all_globals
     has_none = 'None' in code.co_names
@@ -536,19 +550,19 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         code = codeNode.attr
 
     assert iscode(code)
-    code = Code(code, self.scanner, self.currentclass)
+    scanner_code = Code(code, self.scanner, self.currentclass)
 
     # add defaults values to parameter names
     argc = code.co_argcount
-    paramnames = list(code.co_varnames[:argc])
+    paramnames = list(scanner_code.co_varnames[:argc])
 
     # defaults are for last n parameters, thus reverse
     if not 3.0 <= self.version <= 3.1 or self.version >= 3.6:
         paramnames.reverse(); defparams.reverse()
 
     try:
-        ast = self.build_ast(code._tokens,
-                             code._customize,
+        ast = self.build_ast(scanner_code._tokens,
+                             scanner_code._customize,
                              is_lambda = is_lambda,
                              noneInNames = ('None' in code.co_names))
     except (ParserError, ParserError2) as p:
@@ -703,15 +717,22 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
         # docstring exists, dump it
         print_docstring(self, self.indent, code.co_consts[0])
 
-    code._tokens = None # save memory
+    scanner_code._tokens = None # save memory
     assert ast == 'stmts'
 
     all_globals = find_all_globals(ast, set())
-    for g in sorted((all_globals & self.mod_globs) | find_globals(ast, set())):
+    globals, nonlocals = find_globals_and_nonlocals(ast, set(),
+                                                    set(), code, self.version)
+
+    for g in sorted((all_globals & self.mod_globs) | globals):
         self.println(self.indent, 'global ', g)
+
+    for nl in sorted(nonlocals):
+        self.println(self.indent, 'nonlocal ', nl)
+
     self.mod_globs -= all_globals
     has_none = 'None' in code.co_names
     rn = has_none and not find_none(ast)
-    self.gen_source(ast, code.co_name, code._customize, is_lambda=is_lambda,
+    self.gen_source(ast, code.co_name, scanner_code._customize, is_lambda=is_lambda,
                     returnNone=rn)
-    code._tokens = None; code._customize = None # save memory
+    scanner_code._tokens = None; scanner_code._customize = None # save memory

uncompyle6/semantics/pysource.py

@@ -141,7 +141,7 @@ from uncompyle6.semantics.parser_error import ParserError
 from uncompyle6.semantics.check_ast import checker
 from uncompyle6.semantics.customize import customize_for_version
 from uncompyle6.semantics.helper import (
-    print_docstring, find_globals, flatten_list)
+    print_docstring, find_globals_and_nonlocals, flatten_list)
 from uncompyle6.scanners.tok import Token
 
 from uncompyle6.semantics.consts import (
@@ -581,6 +581,7 @@ class SourceWalker(GenericASTTraversal, object):
         self.pending_newlines = 0
         self.params = {
             '_globals': {},
+            '_nonlocals': {},   # Python 3 has nonlocal
             'f': StringIO(),
             'indent': indent,
             'is_lambda': is_lambda,
@@ -1522,6 +1523,13 @@ class SourceWalker(GenericASTTraversal, object):
         # class definition ('class X(A,B,C):')
         cclass = self.currentclass
 
+        # Pick out various needed bits of information
+        # * class_name - the name of the class
+        # * subclass_info - the parameters to the class  e.g.
+        #      class Foo(bar, baz)
+        #             -----------
+        # * subclass_code - the code for the subclass body
+        subclass_info = None
         if self.version > 3.0:
             if node == 'classdefdeco2':
                 if self.version >= 3.6:
@@ -1534,6 +1542,16 @@ class SourceWalker(GenericASTTraversal, object):
             else:
                 build_class = node[0]
                 if self.version >= 3.6:
+                    if build_class == 'build_class_kw':
+                        mkfunc = build_class[1]
+                        assert mkfunc == 'mkfunc'
+                        subclass_info = build_class
+                        if hasattr(mkfunc[0], 'attr') and iscode(mkfunc[0].attr):
+                            subclass_code = mkfunc[0].attr
+                        else:
+                            assert mkfunc[0] == 'load_closure'
+                            subclass_code = mkfunc[1].attr
+                            assert iscode(subclass_code)
                     if build_class[1][0] == 'load_closure':
                         code_node = build_class[1][1]
                     else:
@@ -1580,7 +1598,8 @@ class SourceWalker(GenericASTTraversal, object):
                 else:
                     raise 'Internal Error n_classdef: cannot find class body'
                 if hasattr(build_class[3], '__len__'):
-                    subclass_info = build_class[3]
+                    if not subclass_info:
+                        subclass_info = build_class[3]
                 elif hasattr(build_class[2], '__len__'):
                     subclass_info = build_class[2]
                 else:
@@ -1588,7 +1607,7 @@ class SourceWalker(GenericASTTraversal, object):
             elif self.version >= 3.6 and node == 'classdefdeco2':
                 subclass_info = node
                 subclass_code = build_class[1][0].attr
-            else:
+            elif not subclass_info:
                 subclass_code = build_class[1][0].attr
                 subclass_info = node[0]
         else:
@@ -1663,43 +1682,64 @@ class SourceWalker(GenericASTTraversal, object):
     def print_super_classes3(self, node):
         n = len(node)-1
         if node.kind != 'expr':
+            assert node[n].kind.startswith('CALL_FUNCTION')
+
             kwargs = None
-            # 3.6+ starts having this
             if node[n].kind.startswith('CALL_FUNCTION_KW'):
+                # 3.6+ starts does this
                 kwargs = node[n-1].attr
                 assert isinstance(kwargs, tuple)
-            assert node[n].kind.startswith('CALL_FUNCTION')
-            for i in range(n-2, 0, -1):
-                if not node[i].kind in ['expr', 'LOAD_CLASSNAME']:
-                    break
-                pass
+                i = n - (len(kwargs)+1)
+                j = 1 + n - node[n].attr
+            else:
+                for i in range(n-2, 0, -1):
+                    if not node[i].kind in ['expr', 'LOAD_CLASSNAME']:
+                        break
+                    pass
+
+                if i == n-2:
+                    return
+                i += 2
 
-            if i == n-2:
-                return
             line_separator = ', '
             sep = ''
             self.write('(')
-            j = 0
-            i += 2
             if kwargs:
                 # Last arg is tuple of keyword values: omit
                 l = n - 1
             else:
                 l = n
-            while i < l:
-                # 3.6+ may have this
-                if kwargs:
-                    self.write("%s=" % kwargs[j])
+
+            if kwargs:
+                # 3.6+ does this
+                while j < i:
+                    self.write(sep)
+                    value = self.traverse(node[j])
+                    self.write("%s" % value)
+                    sep = line_separator
                     j += 1
-                value = self.traverse(node[i])
-                i += 1
-                self.write(sep, value)
-                sep = line_separator
-                pass
+
+                j = 0
+                while i < l:
+                    self.write(sep)
+                    value = self.traverse(node[i])
+                    self.write("%s=%s" % (kwargs[j], value))
+                    sep = line_separator
+                    j += 1
+                    i += 1
+            else:
+                while i < l:
+                    value = self.traverse(node[i])
+                    i += 1
+                    self.write(sep, value)
+                    sep = line_separator
+                    pass
             pass
         else:
-            self.write('(')
+            if self.version >= 3.6 and node[0] == 'LOAD_CONST':
+                return
             value = self.traverse(node[0])
+            self.write('(')
             self.write(value)
             pass
 
@@ -2303,11 +2343,16 @@ class SourceWalker(GenericASTTraversal, object):
         # else:
         #    print ast[-1][-1]
 
+        globals, nonlocals = find_globals_and_nonlocals(ast, set(), set(),
+                                                        code, self.version)
         # Add "global" declaration statements at the top
         # of the function
-        for g in sorted(find_globals(ast, set())):
+        for g in sorted(globals):
             self.println(indent, 'global ', g)
 
+        for nl in sorted(nonlocals):
+            self.println(indent, 'nonlocal ', nl)
+
         old_name = self.name
         self.gen_source(ast, code.co_name, code._customize)
         self.name = old_name
@@ -2417,8 +2462,12 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
         'ast': showast,
         'grammar': showgrammar
     }
-    return code_deparse(co, out, version, debug_opts, code_objects, compile_mode,
-                        is_pypy, walker)
+    return code_deparse(co, out,
+                        version=version,
+                        debug_opts=debug_opts,
+                        code_objects=code_objects,
+                        compile_mode=compile_mode,
+                        is_pypy=is_pypy, walker=walker)
 
 def code_deparse(co, out=sys.stdout, version=None, debug_opts=DEFAULT_DEBUG_OPTS,
                  code_objects={}, compile_mode='exec', is_pypy=False, walker=SourceWalker):
@@ -2461,7 +2510,11 @@ def code_deparse(co, out=sys.stdout, version=None, debug_opts=DEFAULT_DEBUG_OPTS
     # save memory
     del tokens
 
-    deparsed.mod_globs = find_globals(deparsed.ast, set())
+    deparsed.mod_globs, nonlocals = find_globals_and_nonlocals(deparsed.ast,
+                                                               set(), set(),
+                                                               co, version)
+
+    assert not nonlocals
 
     # convert leading '__doc__ = "..." into doc string
     try:

uncompyle6/version.py

@@ -12,4 +12,4 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 # This file is suitable for sourcing inside bash as
 # well as importing into Python
-VERSION='3.0.1'
+VERSION='3.1.0'