jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-04 09:22:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-2.4

Browse Source
This commit is contained in:
rocky
2018-04-08 05:27:16 -04:00
parent 19bb16270d ede6eabc40
commit 3116ac8323
34 changed files with 894 additions and 913 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -21,3 +21,4 @@
ChangeLog
__pycache__
build
nohup.out

9
Makefile
View File

@@ -27,19 +27,20 @@ check:
check-short: pytest
$(MAKE) -C test check-short
# Note for 2.6 use <=3.0.1 see requirements-dev.txt
#: Tests for Python 2.7, 3.3 and 3.4
check-2.6 check-2.7 check-3.3 check-3.4: pytest
check-2.6 check-2.7 check-3.3 check-3.4 check-3.5: pytest
$(MAKE) -C test $@
#: Tests for Python 3.2 and 3.5 - pytest doesn't work here
# Or rather 3.5 doesn't work not on Travis
check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
check-3.0 check-3.1 check-3.2 check-3.6:
$(MAKE) -C test $@
check-3.7: pytest
#:Tests for Python 2.6 (doesn't have pytest)
check-2.4 check-2.5 check-2.6:
#:Tests for Python 2.4-2.5 (don't have pytest)
check-2.4 check-2.5:
$(MAKE) -C test $@
#:PyPy 2.6.1 PyPy 5.0.1, or PyPy 5.8.0-beta0

1
__pkginfo__.py
View File

@@ -35,6 +35,7 @@ classifiers = ['Development Status :: 5 - Production/Stable',
'Programming Language :: Python :: 2.5',
'Programming Language :: Python :: 2.6',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.1',
'Programming Language :: Python :: 3.2',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',

2
admin-tools/pyenv-newer-versions
View File

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
echo "This script should be *sourced* rather than run directly through bash"
exit 1
fi
export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.4.8'
export PYVERSIONS='3.5.5 3.6.4 2.6.9 3.3.7 2.7.14 3.2.6 3.1.5 3.4.8'

2
pytest/test_grammar.py
View File

@@ -50,7 +50,7 @@ def test_grammar():
unused_rhs.add('call')
if PYTHON_VERSION == 3.5:
expect_right_recursive.add((('l_stmts',
('lastl_stmt', 'COME_FROM', 'l_stmts'))))
('lastl_stmt', 'come_froms', 'l_stmts'))))
pass
pass
else:

4
requirements-dev.txt
View File

@@ -1,3 +1,3 @@
pytest>=3.0.0
pytest>=3.0.0,<=3.0.1
flake8
hypothesis<=3.8.3
hypothesis<=3.0.0

2
test/Makefile
View File

@@ -37,6 +37,7 @@ check-3.0: check-bytecode
#: Run working tests from Python 3.1
check-3.1: check-bytecode
$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify $(COMPILE)
$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
#: Run working tests from Python 3.2
check-3.2: check-bytecode
@@ -201,6 +202,7 @@ check-bytecode-3.0:
#: Check deparsing Python 3.1
check-bytecode-3.1:
$(PYTHON) test_pythonlib.py --bytecode-3.1 --weak-verify
$(PYTHON) test_pythonlib.py --bytecode-3.1-run --verify-run
#: Check deparsing Python 3.2
check-bytecode-3.2:

BIN
test/bytecode_3.1/02_named_and_kwargs.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.1/04_call_function.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.1_run/05_abc_test.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.2/02_named_and_kwargs.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.2_run/04_call_function.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.2_run/05_abc_test.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.3/02_named_and_kwargs.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.4/02_named_and_kwargs.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.4_run/04_call_function.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.4_run/05_abc_test.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.5_run/04_call_function.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.5_run/07_build_map_unpack.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.6_run/04_call_function.pyc
View File

Binary file not shown.

2
test/grammar-cover/grammar.sh
View File

@@ -30,7 +30,7 @@ while [[ -n $1 ]] ; do
source ./admin-tools/setup-python-2.4.sh
fi
GRAMMAR_TXT=$tmpdir/grammar-${SHORT_VERSION}.txt
pyenv local ${LONG_VERSION}
(cd ../.. && pyenv local ${LONG_VERSION})
cd ./test
if [[ -r $COVER_FILE ]]; then
rm $COVER_FILE

6
test/run-and-email.sh
View File

@@ -14,7 +14,7 @@ function displaytime {
printf '%d seconds\n' $S
}
PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.4.8 2.6.9 3.6.4"}
PYVERSION=${PYVERSION:-"3.5.5 2.7.14 3.2.6 3.3.7 3.4.8 2.6.9 3.6.4"}
# PYVERSION=${PYVERSION:-"3.5.5"}
USER=${USER:-rocky}
@@ -29,7 +29,7 @@ for VERSION in $PYVERSION ; do
if [[ $VERSION == '3.5.5' ]] ; then
MAX_TESTS=224
elif [[ $VERSION == '3.2.6' ]] ; then
MAX_TESTS=75
MAX_TESTS=700
elif [[ $VERSION == '3.6.4' ]] ; then
MAX_TESTS=400
else
@@ -46,7 +46,7 @@ for VERSION in $PYVERSION ; do
rc=$?
echo Python Version $(pyenv local) >> $LOGFILE
echo "" >>LOGFILE
echo "" >>$LOGFILE
typeset -i ALL_FILES_ENDTIME=$(date +%s)
(( time_diff = ALL_FILES_ENDTIME - ALL_FILES_STARTTIME))

18
test/simple_source/bug31/05_abc_test.py Normal file
View File

@@ -0,0 +1,18 @@
# Extracted from Python 3.5 test_abc.py
# Bug is class having only a single kwarg
# subclass.
import abc
import unittest
from inspect import isabstract
def test_abstractmethod_integration(self):
for abstractthing in [abc.abstractmethod]:
class C(metaclass=abc.ABCMeta):
@abstractthing
def foo(self): pass # abstract
def bar(self): pass # concrete
assert C.__abstractmethods__, {"foo"}
assert isabstract(C)
pass
test_abstractmethod_integration(None)

4
test/simple_source/bug33/02_named_and_kwargs.py
View File

@@ -1,5 +1,6 @@
# Python 3.6 subprocess.py bug
# Bug is getting params correct: timeout before **kwargs
import subprocess
def call(*popenargs, timeout=None, **kwargs):
return
@@ -14,6 +15,9 @@ def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE,
# From 3.4 asyncio/locks.py
# Bug was handling" "value=1, *"
class Semaphore:
pass
class BoundedSemaphore(Semaphore):
def __init__(self, value=1, *, loop=None):
super().__init__(value, loop=loop)

18
test/simple_source/bug35/04_call_function.py
View File

@@ -1,5 +1,6 @@
# From sql/schema.py and 3.5 _strptime.py
# Note that kwargs comes before "positional" args
# Bug was code not knowing which Python versions
# have kwargs coming before positional args in code.
# RUNNABLE!
@@ -63,3 +64,18 @@ def unpack_archive(func, filename, dict, format_info, extract_dir=None):
import xdrlib
def assertRaisesConversion(self, *args):
self.assertRaises(xdrlib.ConversionError, *args)
# From 3.2.6 _pyio.py
class BlockingIOError(IOError):
def __init__(self, errno, strerror, characters_written=5):
super().__init__(errno, strerror)
# From urllib/parse.py
# Bug was using a subclass made from a call (to namedtuple)
from collections import namedtuple
class ResultMixin(object):
pass
class SplitResult(namedtuple('SplitResult', 'scheme netloc path query fragment'), ResultMixin):
pass

10
test/simple_source/bug35/07_build_map_unpack.py Normal file
View File

@@ -0,0 +1,10 @@
# Adapted from Python 3.6 trace.py
# Bug was in handling BUID_TUPLE_UNPACK created via
# *opts.arguments
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('filename', nargs='?')
parser.add_argument('arguments', nargs=argparse.REMAINDER)
opts = parser.parse_args(["foo", "a", "b"])
argv = opts.filename, *opts.arguments
assert argv == ('foo', 'a', 'b')

2
test/stdlib/run-and-email.sh
View File

@@ -2,7 +2,7 @@
USER=${USER:-rocky}
EMAIL=${EMAIL:-rb@dustyfeet.com}
SUBJECT_PREFIX="stdlib unit testing for"
for VERSION in 3.4.8 3.5.5 3.6.4 ; do
for VERSION in 2.6.9 2.7.14 3.4.8 3.5.5 3.6.4 ; do
typeset -i rc=0
LOGFILE=/tmp/runtests-$VERSION-$$.log
if ! pyenv local $VERSION ; then

2
test/stdlib/runtests.sh
View File

@@ -101,8 +101,10 @@ case $PYVERSION in
3.5)
SKIP_TESTS=(
[test_decorators.py]=1 # Control flow wrt "if elif"
[test_quopri.py]=1 # Fails in crontab environment?
)
;;
3.6)
SKIP_TESTS=(
[test_contains.py]=1 # Code "while False: yield None" is optimized away in compilation

25
uncompyle6/parsers/parse3.py
View File

@@ -117,9 +117,14 @@ class Python3Parser(PythonParser):
classdef ::= build_class store
# FIXME: we need to add these because don't detect this properly
# in custom rules. Specifically if one of the exprs is CALL_FUNCTION
# then we'll mistake that for the final CALL_FUNCTION.
# We can fix by triggering on the CALL_FUNCTION op
# Python3 introduced LOAD_BUILD_CLASS
# Other definitions are in a custom rule
build_class ::= LOAD_BUILD_CLASS mkfunc expr call CALL_FUNCTION_3
build_class ::= LOAD_BUILD_CLASS mkfunc expr call expr CALL_FUNCTION_4
stmt ::= classdefdeco
classdefdeco ::= classdefdeco1 store
@@ -424,7 +429,7 @@ class Python3Parser(PythonParser):
LOAD_CONST CALL_FUNCTION_n
build_class ::= LOAD_BUILD_CLASS mkfunc
expr
call_function
call
CALL_FUNCTION_3
'''
# FIXME: I bet this can be simplified
@@ -627,7 +632,7 @@ class Python3Parser(PythonParser):
self.addRule(rule, nop_func)
rule = 'expr ::= build_list_unpack'
self.addRule(rule, nop_func)
elif opname_base == 'BUILD_MAP':
elif opname_base in ('BUILD_MAP', 'BUILD_MAP_UNPACK'):
kvlist_n = "kvlist_%s" % token.attr
if opname == 'BUILD_MAP_n':
# PyPy sometimes has no count. Sigh.
@@ -646,23 +651,21 @@ class Python3Parser(PythonParser):
# FIXME: Use the attr
# so this doesn't run into exponential parsing time.
if opname.startswith('BUILD_MAP_UNPACK'):
# FIXME: start here
# rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
self.add_unique_rule(rule, opname, token.attr, customize)
rule = 'dict_entry ::= ' + 'expr ' * (token.attr*2)
self.add_unique_rule(rule, opname, token.attr, customize)
rule = 'dict ::= %s' % ('dict_entry ' * token.attr)
self.addRule(rule, nop_func)
# FIXME: start here. The LHS should be unmap_dict, not dict.
# FIXME: really we need a combination of dict_entry-like things.
# It just so happens the most common case is not to mix
# dictionary comphensions with dictionary, elements
if self.seen_LOAD_DICTCOMP:
rule = 'dict ::= %s%s' % ('dict_comp ' * token.attr, opname)
self.addRule(rule, nop_func)
rule = 'unmap_dict ::= %s%s' % (('dict ' * token.attr), opname)
rule = """
expr ::= unmap_dict
unmap_dict ::= %s%s
""" % ('expr ' * token.attr, opname)
else:
rule = "%s ::= %s %s" % (kvlist_n, 'expr ' * (token.attr*2), opname)
self.add_unique_rule(rule, opname, token.attr, customize)
@@ -680,7 +683,9 @@ class Python3Parser(PythonParser):
v = token.attr
rule = ('starred ::= %s %s' % ('expr ' * v, opname))
self.addRule(rule, nop_func)
elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'):
elif opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE',
'BUILD_TUPLE_UNPACK'):
v = token.attr
is_LOAD_CLOSURE = False

6
uncompyle6/scanners/scanner3.py
View File

@@ -136,6 +136,7 @@ class Scanner3(Scanner):
(self.opc.JUMP_FORWARD,),
(self.opc.JUMP_ABSOLUTE,)]
# FIXME: remove this and use instead info from xdis.
# Opcodes that take a variable number of arguments
# (expr's)
varargs_ops = set([
@@ -146,6 +147,11 @@ class Scanner3(Scanner):
if is_pypy:
varargs_ops.add(self.opc.CALL_METHOD)
if self.version >= 3.5:
varargs_ops |= set([self.opc.BUILD_SET_UNPACK,
self.opc.BUILD_MAP_UNPACK, # we will handle this later
self.opc.BUILD_LIST_UNPACK,
self.opc.BUILD_TUPLE_UNPACK])
if self.version >= 3.6:
varargs_ops.add(self.opc.BUILD_CONST_KEY_MAP)
# Below is in bit order, "default = bit 0, closure = bit 3

643
uncompyle6/semantics/customize.py
View File

@@ -17,18 +17,10 @@
"""
from uncompyle6.semantics.consts import (
PRECEDENCE, INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT)
TABLE_R, TABLE_DIRECT)
from uncompyle6.semantics.make_function import (
make_function3_annotate,
)
from xdis.util import COMPILER_FLAG_BIT
from xdis.code import iscode
from uncompyle6.parsers.astnode import AST
from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.helper import flatten_list
from spark_parser.ast import GenericASTTraversalPruningException
def customize_for_version(self, is_pypy, version):
if is_pypy:
@@ -189,635 +181,6 @@ def customize_for_version(self, is_pypy, version):
})
if version >= 3.0:
TABLE_DIRECT.update({
'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
})
def n_mkfunc_annotate(node):
if self.version >= 3.3 or node[-2] == 'kwargs':
# LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3
# EXTENDED_ARG
# MAKE_FUNCTION ..
code = node[-4]
elif node[-3] == 'expr':
code = node[-3][0]
else:
# LOAD_CONST code object ..
# MAKE_FUNCTION ..
code = node[-3]
self.indent_more()
for annotate_last in range(len(node)-1, -1, -1):
if node[annotate_last] == 'annotate_tuple':
break
# FIXME: the real situation is that when derived from
# function_def_annotate we the name has been filled in.
# But when derived from funcdefdeco it hasn't Would like a better
# way to distinquish.
if self.f.getvalue()[-4:] == 'def ':
self.write(code.attr.co_name)
# FIXME: handle and pass full annotate args
make_function3_annotate(self, node, is_lambda=False,
codeNode=code, annotate_last=annotate_last)
if len(self.param_stack) > 1:
self.write('\n\n')
else:
self.write('\n\n\n')
self.indent_less()
self.prune() # stop recursing
self.n_mkfunc_annotate = n_mkfunc_annotate
if version >= 3.4:
########################
# Python 3.4+ Additions
#######################
TABLE_DIRECT.update({
'LOAD_CLASSDEREF': ( '%{pattr}', ),
})
if version == 3.4:
def n_call(node):
mapping = self._get_mapping(node)
key = node
for i in mapping[1:]:
key = key[i]
pass
if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
# We may want to fill this in...
# But it is distinct from CALL_FUNCTION_VAR below
pass
elif key.kind.startswith('CALL_FUNCTION_VAR'):
# CALL_FUNCTION_VAR's top element of the stack contains
# the variable argument list, then comes
# annotation args, then keyword args.
# In the most least-top-most stack entry, but position 1
# in node order, the positional args.
argc = node[-1].attr
nargs = argc & 0xFF
kwargs = (argc >> 8) & 0xFF
# FIXME: handle annotation args
if kwargs != 0:
# kwargs == 0 is handled by the table entry
# Should probably handle it here though.
if nargs == 0:
template = ('%c(*%c, %C)',
0, -2, (1, kwargs+1, ', '))
else:
template = ('%c(%C, *%c, %C)',
0, (1, nargs+1, ', '),
-2, (-2-kwargs, -2, ', '))
self.template_engine(template, node)
self.prune()
self.default(node)
self.n_call = n_call
########################
# Python 3.5+ Additions
#######################
if version >= 3.5:
TABLE_DIRECT.update({
'await_expr': ( 'await %c', 0),
'await_stmt': ( '%|%c\n', 0),
'async_for_stmt': (
'%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
'async_forelse_stmt': (
'%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
'async_with_stmt': (
'%|async with %c:\n%+%c%-', 0, 7),
'async_with_as_stmt': (
'%|async with %c as %c:\n%+%c%-', 0, 6, 7),
'unmap_dict': ( '{**%C}', (0, -1, ', **') ),
# 'unmapexpr': ( '{**%c}', 0), # done by n_unmapexpr
})
def async_call(node):
self.f.write('async ')
node.kind == 'call'
p = self.prec
self.prec = 80
self.template_engine(('%c(%P)', 0, (1, -4, ', ',
100)), node)
self.prec = p
node.kind == 'async_call'
self.prune()
self.n_async_call = async_call
self.n_build_list_unpack = self.n_list
if version == 3.5:
def n_call(node):
mapping = self._get_mapping(node)
table = mapping[0]
key = node
for i in mapping[1:]:
key = key[i]
pass
if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
# Python 3.5 changes the stack position of
# *args: kwargs come after *args whereas
# in earlier Pythons, *args is at the end
# which simplifies things from our
# perspective. Python 3.6+ replaces
# CALL_FUNCTION_VAR_KW with
# CALL_FUNCTION_EX We will just swap the
# order to make it look like earlier
# Python 3.
entry = table[key.kind]
kwarg_pos = entry[2][1]
args_pos = kwarg_pos - 1
# Put last node[args_pos] after subsequent kwargs
while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
# swap node[args_pos] with node[kwargs_pos]
node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
args_pos = kwarg_pos
kwarg_pos += 1
elif key.kind.startswith('CALL_FUNCTION_VAR'):
# CALL_FUNCTION_VAR's top element of the stack contains
# the variable argument list, then comes
# annotation args, then keyword args.
# In the most least-top-most stack entry, but position 1
# in node order, the positional args.
argc = node[-1].attr
nargs = argc & 0xFF
kwargs = (argc >> 8) & 0xFF
# FIXME: handle annotation args
if nargs > 0:
template = ('%c(%C, ', 0, (1, nargs+1, ', '))
else:
template = ('%c(', 0)
self.template_engine(template, node)
args_node = node[-2]
if args_node in ('pos_arg', 'expr'):
args_node = args_node[0]
if args_node == 'build_list_unpack':
template = ('*%P)', (0, len(args_node)-1, ', *', 100))
self.template_engine(template, args_node)
else:
if len(node) - nargs > 3:
template = ('*%c, %C)', nargs+1, (nargs+kwargs+1, -1, ', '))
else:
template = ('*%c)', nargs+1)
self.template_engine(template, node)
self.prune()
self.default(node)
self.n_call = n_call
def n_function_def(node):
if self.version == 3.6:
code_node = node[0][0]
else:
code_node = node[0][1]
is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
if (is_code and
(code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
self.template_engine(('\n\n%|async def %c\n',
-2), node)
else:
self.template_engine(('\n\n%|def %c\n', -2),
node)
self.prune()
self.n_function_def = n_function_def
def unmapexpr(node):
last_n = node[0][-1]
for n in node[0]:
self.preorder(n)
if n != last_n:
self.f.write(', **')
pass
pass
self.prune()
pass
self.n_unmapexpr = unmapexpr
if version >= 3.6:
########################
# Python 3.6+ Additions
#######################
# Value 100 is important; it is exactly
# module/function precidence.
PRECEDENCE['call_kw'] = 100
PRECEDENCE['call_kw36'] = 100
PRECEDENCE['call_ex'] = 100
PRECEDENCE['call_ex_kw'] = 100
PRECEDENCE['call_ex_kw2'] = 100
PRECEDENCE['call_ex_kw3'] = 100
PRECEDENCE['call_ex_kw4'] = 100
TABLE_DIRECT.update({
'tryfinally36': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
(1, 'returns'), 3 ),
'fstring_expr': ( "{%c%{conversion}}", 0),
# FIXME: the below assumes the format strings
# don't have ''' in them. Fix this properly
'fstring_single': ( "f'''{%c%{conversion}}'''", 0),
'fstring_multi': ( "f'''%c'''", 0),
'func_args36': ( "%c(**", 0),
'try_except36': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
'except_return': ( '%|except:\n%+%c%-', 3 ),
'unpack_list': ( '*%c', (0, 'list') ),
'call_ex' : (
'%c(%p)',
(0, 'expr'), (1, 100)),
'call_ex_kw' : (
'%c(%p)',
(0, 'expr'), (2, 100)),
})
TABLE_R.update({
'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)),
# Not quite right
'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')),
})
def build_unpack_tuple_with_call(node):
if node[0] == 'expr':
tup = node[0][0]
else:
tup = node[0]
pass
assert tup == 'tuple'
self.call36_tuple(tup)
buwc = node[-1]
assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
for n in node[1:-1]:
self.f.write(', *')
self.preorder(n)
pass
self.prune()
return
self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call
def build_unpack_map_with_call(node):
n = node[0]
if n == 'expr':
n = n[0]
if n == 'dict':
self.call36_dict(n)
first = 1
sep = ', **'
else:
first = 0
sep = '**'
for n in node[first:-1]:
self.f.write(sep)
self.preorder(n)
sep = ', **'
pass
self.prune()
return
self.n_build_map_unpack_with_call = build_unpack_map_with_call
def call_ex_kw2(node):
"""Handle CALL_FUNCTION_EX 2 (have KW) but with
BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
# This is weird shit. Thanks Python!
self.preorder(node[0])
self.write('(')
assert node[1] == 'build_tuple_unpack_with_call'
btuwc = node[1]
tup = btuwc[0]
if tup == 'expr':
tup = tup[0]
assert tup == 'tuple'
self.call36_tuple(tup)
assert node[2] == 'build_map_unpack_with_call'
self.write(', ')
d = node[2][0]
if d == 'expr':
d = d[0]
assert d == 'dict'
self.call36_dict(d)
args = btuwc[1]
self.write(', *')
self.preorder(args)
self.write(', **')
star_star_args = node[2][1]
if star_star_args == 'expr':
star_star_args = star_star_args[0]
self.preorder(star_star_args)
self.write(')')
self.prune()
self.n_call_ex_kw2 = call_ex_kw2
def call_ex_kw3(node):
"""Handle CALL_FUNCTION_EX 1 (have KW) but without
BUILD_MAP_UNPACK_WITH_CALL"""
self.preorder(node[0])
self.write('(')
args = node[1][0]
if args == 'expr':
args = args[0]
if args == 'tuple':
if self.call36_tuple(args) > 0:
self.write(', ')
pass
pass
self.write('*')
self.preorder(node[1][1])
self.write(', ')
kwargs = node[2]
if kwargs == 'expr':
kwargs = kwargs[0]
if kwargs == 'dict':
self.call36_dict(kwargs)
else:
self.write('**')
self.preorder(kwargs)
self.write(')')
self.prune()
self.n_call_ex_kw3 = call_ex_kw3
def call_ex_kw4(node):
"""Handle CALL_FUNCTION_EX {1 or 2} but without
BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
self.preorder(node[0])
self.write('(')
args = node[1][0]
if args == 'tuple':
if self.call36_tuple(args) > 0:
self.write(', ')
pass
pass
else:
self.write('*')
self.preorder(args)
self.write(', ')
pass
kwargs = node[2]
if kwargs == 'expr':
kwargs = kwargs[0]
call_function_ex = node[-1]
assert call_function_ex == 'CALL_FUNCTION_EX_KW'
# FIXME: decide if the below test be on kwargs == 'dict'
if (call_function_ex.attr & 1 and
(not isinstance(kwargs, Token) and kwargs != 'attribute')
and not kwargs[0].kind.startswith('kvlist')):
self.call36_dict(kwargs)
else:
self.write('**')
self.preorder(kwargs)
self.write(')')
self.prune()
self.n_call_ex_kw4 = call_ex_kw4
def call36_tuple(node):
"""
A tuple used in a call, these are like normal tuples but they
don't have the enclosing parenthesis.
"""
assert node == 'tuple'
# Note: don't iterate over last element which is a
# BUILD_TUPLE...
flat_elems = flatten_list(node[:-1])
self.indent_more(INDENT_PER_LEVEL)
sep = ''
for elem in flat_elems:
if elem in ('ROT_THREE', 'EXTENDED_ARG'):
continue
assert elem == 'expr'
line_number = self.line_number
value = self.traverse(elem)
if line_number != self.line_number:
sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
self.write(sep, value)
sep = ', '
self.indent_less(INDENT_PER_LEVEL)
return len(flat_elems)
self.call36_tuple = call36_tuple
def call36_dict(node):
"""
A dict used in a call_ex_kw2, which are a dictionary items expressed
in a call. This should format to:
a=1, b=2
In other words, no braces, no quotes around keys and ":" becomes
"=".
We will source-code use line breaks to guide us when to break.
"""
p = self.prec
self.prec = 100
self.indent_more(INDENT_PER_LEVEL)
sep = INDENT_PER_LEVEL[:-1]
line_number = self.line_number
if node[0].kind.startswith('kvlist'):
# Python 3.5+ style key/value list in dict
kv_node = node[0]
l = list(kv_node)
i = 0
length = len(l)
# FIXME: Parser-speed improved grammars will have BUILD_MAP
# at the end. So in the future when everything is
# complete, we can do an "assert" instead of "if".
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
# Respect line breaks from source
while i < length:
self.write(sep)
name = self.traverse(l[i], indent='')
# Strip off beginning and trailing quotes in name
name = name[1:-1]
if i > 0:
line_number = self.indent_if_source_nl(line_number,
self.indent + INDENT_PER_LEVEL[:-1])
line_number = self.line_number
self.write(name, '=')
value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
self.write(value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
line_number = self.line_number
i += 2
pass
elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
keys_node = node[-2]
keys = keys_node.attr
# from trepan.api import debug; debug()
assert keys_node == 'LOAD_CONST' and isinstance(keys, tuple)
for i in range(node[-1].attr):
self.write(sep)
self.write(keys[i], '=')
value = self.traverse(node[i], indent='')
self.write(value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
line_number = self.line_number
pass
pass
else:
self.write("**")
try:
self.default(node)
except GenericASTTraversalPruningException:
pass
self.prec = p
self.indent_less(INDENT_PER_LEVEL)
return
self.call36_dict = call36_dict
FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
def n_except_suite_finalize(node):
if node[1] == 'returns' and self.hide_internal:
# Process node[1] only.
# The code after "returns", e.g. node[3], is dead code.
# Adding it is wrong as it dedents and another
# exception handler "except_stmt" afterwards.
# Note it is also possible that the grammar is wrong here.
# and this should not be "except_stmt".
self.indent_more()
self.preorder(node[1])
self.indent_less()
else:
self.default(node)
self.prune()
self.n_except_suite_finalize = n_except_suite_finalize
def n_formatted_value(node):
if node[0] == 'LOAD_CONST':
self.write(node[0].attr)
self.prune()
else:
self.default(node)
self.n_formatted_value = n_formatted_value
def f_conversion(node):
node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '')
def fstring_expr(node):
f_conversion(node)
self.default(node)
self.n_fstring_expr = fstring_expr
def fstring_single(node):
f_conversion(node)
self.default(node)
self.n_fstring_single = fstring_single
# def kwargs_only_36(node):
# keys = node[-1].attr
# num_kwargs = len(keys)
# values = node[:num_kwargs]
# for i, (key, value) in enumerate(zip(keys, values)):
# self.write(key + '=')
# self.preorder(value)
# if i < num_kwargs:
# self.write(',')
# self.prune()
# return
# self.n_kwargs_only_36 = kwargs_only_36
def n_call_kw36(node):
self.template_engine(("%c(", 0), node)
keys = node[-2].attr
num_kwargs = len(keys)
num_posargs = len(node) - (num_kwargs + 2)
n = len(node)
assert n >= len(keys)+1, \
'not enough parameters keyword-tuple values'
sep = ''
line_number = self.line_number
for i in range(1, num_posargs):
self.write(sep)
self.preorder(node[i])
if line_number != self.line_number:
sep = ",\n" + self.indent + " "
else:
sep = ", "
line_number = self.line_number
i = num_posargs
j = 0
# FIXME: adjust output for line breaks?
while i < n-2:
self.write(sep)
self.write(keys[j] + '=')
self.preorder(node[i])
if line_number != self.line_number:
sep = ",\n" + self.indent + " "
else:
sep = ", "
i += 1
j += 1
self.write(')')
self.prune()
return
self.n_call_kw36 = n_call_kw36
def starred(node):
l = len(node)
assert l > 0
pos_args = node[0]
if pos_args == 'expr':
pos_args = pos_args[0]
if pos_args == 'tuple':
build_tuple = pos_args[0]
if build_tuple.kind.startswith('BUILD_TUPLE'):
tuple_len = 0
else:
tuple_len = len(node) - 1
star_start = 1
template = '%C', (0, -1, ', ')
self.template_engine(template, pos_args)
if tuple_len == 0:
self.write("*()")
# That's it
self.prune()
self.write(', ')
else:
star_start = 0
if l > 1:
template = ( '*%C', (star_start, -1, ', *') )
else:
template = ( '*%c', (star_start, 'expr') )
self.template_engine(template, node)
self.prune()
self.n_starred = starred
def return_closure(node):
# Nothing should be output here
self.prune()
return
self.n_return_closure = return_closure
pass # version >= 3.6
pass # version >= 3.4
pass # version >= 3.0
from uncompyle6.semantics.customize3 import customize_for_version3
customize_for_version3(self, version)
return

759
uncompyle6/semantics/customize3.py Normal file
View File

@@ -0,0 +1,759 @@
# Copyright (c) 2018 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Isolate Python 3 version-specific semantic actions here.
"""
from uncompyle6.semantics.consts import (
INDENT_PER_LEVEL, PRECEDENCE, TABLE_DIRECT, TABLE_R)
from xdis.code import iscode
from xdis.util import COMPILER_FLAG_BIT
from spark_parser.ast import GenericASTTraversalPruningException
from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.helper import flatten_list
from uncompyle6.semantics.make_function import make_function3_annotate
def customize_for_version3(self, version):
TABLE_DIRECT.update({
'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
})
if version >= 3.3:
def n_yield_from(node):
self.write('yield from')
self.write(' ')
if 3.3 <= self.version <= 3.4:
self.preorder(node[0][0][0][0])
elif self.version >= 3.5:
self.preorder(node[0])
else:
assert False, "dunno about this python version"
self.prune() # stop recursing
self.n_yield_from = n_yield_from
if 3.2 <= version <= 3.4:
def n_call(node):
mapping = self._get_mapping(node)
key = node
for i in mapping[1:]:
key = key[i]
pass
if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
# We may want to fill this in...
# But it is distinct from CALL_FUNCTION_VAR below
pass
elif key.kind.startswith('CALL_FUNCTION_VAR'):
# CALL_FUNCTION_VAR's top element of the stack contains
# the variable argument list, then comes
# annotation args, then keyword args.
# In the most least-top-most stack entry, but position 1
# in node order, the positional args.
argc = node[-1].attr
nargs = argc & 0xFF
kwargs = (argc >> 8) & 0xFF
# FIXME: handle annotation args
if kwargs != 0:
# kwargs == 0 is handled by the table entry
# Should probably handle it here though.
if nargs == 0:
template = ('%c(*%c, %C)',
0, -2, (1, kwargs+1, ', '))
else:
template = ('%c(%C, *%c, %C)',
0, (1, nargs+1, ', '),
-2, (-2-kwargs, -2, ', '))
self.template_engine(template, node)
self.prune()
self.default(node)
self.n_call = n_call
def n_mkfunc_annotate(node):
if self.version >= 3.3 or node[-2] == 'kwargs':
# LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3
# EXTENDED_ARG
# MAKE_FUNCTION ..
code = node[-4]
elif node[-3] == 'expr':
code = node[-3][0]
else:
# LOAD_CONST code object ..
# MAKE_FUNCTION ..
code = node[-3]
self.indent_more()
for annotate_last in range(len(node)-1, -1, -1):
if node[annotate_last] == 'annotate_tuple':
break
# FIXME: the real situation is that when derived from
# function_def_annotate we the name has been filled in.
# But when derived from funcdefdeco it hasn't Would like a better
# way to distinquish.
if self.f.getvalue()[-4:] == 'def ':
self.write(code.attr.co_name)
# FIXME: handle and pass full annotate args
make_function3_annotate(self, node, is_lambda=False,
codeNode=code, annotate_last=annotate_last)
if len(self.param_stack) > 1:
self.write('\n\n')
else:
self.write('\n\n\n')
self.indent_less()
self.prune() # stop recursing
self.n_mkfunc_annotate = n_mkfunc_annotate
if version >= 3.4:
########################
# Python 3.4+ Additions
#######################
TABLE_DIRECT.update({
'LOAD_CLASSDEREF': ( '%{pattr}', ),
})
########################
# Python 3.5+ Additions
#######################
if version >= 3.5:
TABLE_DIRECT.update({
'await_expr': ( 'await %c', 0),
'await_stmt': ( '%|%c\n', 0),
'async_for_stmt': (
'%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
'async_forelse_stmt': (
'%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
'async_with_stmt': (
'%|async with %c:\n%+%c%-', 0, 7),
'async_with_as_stmt': (
'%|async with %c as %c:\n%+%c%-', 0, 6, 7),
'unmap_dict': ( '{**%C}', (0, -1, ', **') ),
# 'unmapexpr': ( '{**%c}', 0), # done by n_unmapexpr
})
def async_call(node):
self.f.write('async ')
node.kind == 'call'
p = self.prec
self.prec = 80
self.template_engine(('%c(%P)', 0, (1, -4, ', ',
100)), node)
self.prec = p
node.kind == 'async_call'
self.prune()
self.n_async_call = async_call
self.n_build_list_unpack = self.n_list
if version == 3.5:
def n_call(node):
mapping = self._get_mapping(node)
table = mapping[0]
key = node
for i in mapping[1:]:
key = key[i]
pass
if key.kind.startswith('CALL_FUNCTION_VAR_KW'):
# Python 3.5 changes the stack position of
# *args: kwargs come after *args whereas
# in earlier Pythons, *args is at the end
# which simplifies things from our
# perspective. Python 3.6+ replaces
# CALL_FUNCTION_VAR_KW with
# CALL_FUNCTION_EX We will just swap the
# order to make it look like earlier
# Python 3.
entry = table[key.kind]
kwarg_pos = entry[2][1]
args_pos = kwarg_pos - 1
# Put last node[args_pos] after subsequent kwargs
while node[kwarg_pos] == 'kwarg' and kwarg_pos < len(node):
# swap node[args_pos] with node[kwargs_pos]
node[kwarg_pos], node[args_pos] = node[args_pos], node[kwarg_pos]
args_pos = kwarg_pos
kwarg_pos += 1
elif key.kind.startswith('CALL_FUNCTION_VAR'):
# CALL_FUNCTION_VAR's top element of the stack contains
# the variable argument list, then comes
# annotation args, then keyword args.
# In the most least-top-most stack entry, but position 1
# in node order, the positional args.
argc = node[-1].attr
nargs = argc & 0xFF
kwargs = (argc >> 8) & 0xFF
# FIXME: handle annotation args
if nargs > 0:
template = ('%c(%C, ', 0, (1, nargs+1, ', '))
else:
template = ('%c(', 0)
self.template_engine(template, node)
args_node = node[-2]
if args_node in ('pos_arg', 'expr'):
args_node = args_node[0]
if args_node == 'build_list_unpack':
template = ('*%P)', (0, len(args_node)-1, ', *', 100))
self.template_engine(template, args_node)
else:
if len(node) - nargs > 3:
template = ('*%c, %C)', nargs+1, (nargs+kwargs+1, -1, ', '))
else:
template = ('*%c)', nargs+1)
self.template_engine(template, node)
self.prune()
self.default(node)
self.n_call = n_call
def n_function_def(node):
if self.version == 3.6:
code_node = node[0][0]
else:
code_node = node[0][1]
is_code = hasattr(code_node, 'attr') and iscode(code_node.attr)
if (is_code and
(code_node.attr.co_flags & COMPILER_FLAG_BIT['COROUTINE'])):
self.template_engine(('\n\n%|async def %c\n',
-2), node)
else:
self.template_engine(('\n\n%|def %c\n', -2),
node)
self.prune()
self.n_function_def = n_function_def
def unmapexpr(node):
last_n = node[0][-1]
for n in node[0]:
self.preorder(n)
if n != last_n:
self.f.write(', **')
pass
pass
self.prune()
pass
self.n_unmapexpr = unmapexpr
# FIXME: start here
def n_list_unpack(node):
"""
prettyprint an unpacked list or tuple
"""
p = self.prec
self.prec = 100
lastnode = node.pop()
lastnodetype = lastnode.kind
# If this build list is inside a CALL_FUNCTION_VAR,
# then the first * has already been printed.
# Until I have a better way to check for CALL_FUNCTION_VAR,
# will assume that if the text ends in *.
last_was_star = self.f.getvalue().endswith('*')
if lastnodetype.startswith('BUILD_LIST'):
self.write('['); endchar = ']'
elif lastnodetype.startswith('BUILD_TUPLE'):
# Tuples can appear places that can NOT
# have parenthesis around them, like array
# subscripts. We check for that by seeing
# if a tuple item is some sort of slice.
no_parens = False
for n in node:
if n == 'expr' and n[0].kind.startswith('build_slice'):
no_parens = True
break
pass
if no_parens:
endchar = ''
else:
self.write('('); endchar = ')'
pass
elif lastnodetype.startswith('BUILD_SET'):
self.write('{'); endchar = '}'
elif lastnodetype.startswith('BUILD_MAP_UNPACK'):
self.write('{*'); endchar = '}'
elif lastnodetype.startswith('ROT_TWO'):
self.write('('); endchar = ')'
else:
raise TypeError('Internal Error: n_build_list expects list, tuple, set, or unpack')
flat_elems = flatten_list(node)
self.indent_more(INDENT_PER_LEVEL)
sep = ''
for elem in flat_elems:
if elem in ('ROT_THREE', 'EXTENDED_ARG'):
continue
assert elem == 'expr'
line_number = self.line_number
value = self.traverse(elem)
if elem[0] == 'tuple':
assert value[0] == '('
assert value[-1] == ')'
value = value[1:-1]
if value[-1] == ',':
# singleton tuple
value = value[:-1]
else:
value = '*' + value
if line_number != self.line_number:
sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
else:
if sep != '': sep += ' '
if not last_was_star:
pass
else:
last_was_star = False
self.write(sep, value)
sep = ','
if lastnode.attr == 1 and lastnodetype.startswith('BUILD_TUPLE'):
self.write(',')
self.write(endchar)
self.indent_less(INDENT_PER_LEVEL)
self.prec = p
self.prune()
return
self.n_tuple_unpack = n_list_unpack
if version >= 3.6:
########################
# Python 3.6+ Additions
#######################
# Value 100 is important; it is exactly
# module/function precidence.
PRECEDENCE['call_kw'] = 100
PRECEDENCE['call_kw36'] = 100
PRECEDENCE['call_ex'] = 100
PRECEDENCE['call_ex_kw'] = 100
PRECEDENCE['call_ex_kw2'] = 100
PRECEDENCE['call_ex_kw3'] = 100
PRECEDENCE['call_ex_kw4'] = 100
PRECEDENCE['unmap_dict'] = 0
TABLE_DIRECT.update({
'tryfinally36': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n',
(1, 'returns'), 3 ),
'fstring_expr': ( "{%c%{conversion}}", 0),
# FIXME: the below assumes the format strings
# don't have ''' in them. Fix this properly
'fstring_single': ( "f'''{%c%{conversion}}'''", 0),
'fstring_multi': ( "f'''%c'''", 0),
'func_args36': ( "%c(**", 0),
'try_except36': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
'except_return': ( '%|except:\n%+%c%-', 3 ),
'unpack_list': ( '*%c', (0, 'list') ),
'call_ex' : (
'%c(%p)',
(0, 'expr'), (1, 100)),
'call_ex_kw' : (
'%c(%p)',
(0, 'expr'), (2, 100)),
})
TABLE_R.update({
'CALL_FUNCTION_EX': ('%c(*%P)', 0, (1, 2, ', ', 100)),
# Not quite right
'CALL_FUNCTION_EX_KW': ('%c(**%C)', 0, (2, 3, ',')),
})
def build_unpack_tuple_with_call(node):
if node[0] == 'expr':
tup = node[0][0]
else:
tup = node[0]
pass
assert tup == 'tuple'
self.call36_tuple(tup)
buwc = node[-1]
assert buwc.kind.startswith('BUILD_TUPLE_UNPACK_WITH_CALL')
for n in node[1:-1]:
self.f.write(', *')
self.preorder(n)
pass
self.prune()
return
self.n_build_tuple_unpack_with_call = build_unpack_tuple_with_call
def build_unpack_map_with_call(node):
n = node[0]
if n == 'expr':
n = n[0]
if n == 'dict':
self.call36_dict(n)
first = 1
sep = ', **'
else:
first = 0
sep = '**'
for n in node[first:-1]:
self.f.write(sep)
self.preorder(n)
sep = ', **'
pass
self.prune()
return
self.n_build_map_unpack_with_call = build_unpack_map_with_call
def call_ex_kw2(node):
"""Handle CALL_FUNCTION_EX 2 (have KW) but with
BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
# This is weird shit. Thanks Python!
self.preorder(node[0])
self.write('(')
assert node[1] == 'build_tuple_unpack_with_call'
btuwc = node[1]
tup = btuwc[0]
if tup == 'expr':
tup = tup[0]
assert tup == 'tuple'
self.call36_tuple(tup)
assert node[2] == 'build_map_unpack_with_call'
self.write(', ')
d = node[2][0]
if d == 'expr':
d = d[0]
assert d == 'dict'
self.call36_dict(d)
args = btuwc[1]
self.write(', *')
self.preorder(args)
self.write(', **')
star_star_args = node[2][1]
if star_star_args == 'expr':
star_star_args = star_star_args[0]
self.preorder(star_star_args)
self.write(')')
self.prune()
self.n_call_ex_kw2 = call_ex_kw2
def call_ex_kw3(node):
"""Handle CALL_FUNCTION_EX 1 (have KW) but without
BUILD_MAP_UNPACK_WITH_CALL"""
self.preorder(node[0])
self.write('(')
args = node[1][0]
if args == 'expr':
args = args[0]
if args == 'tuple':
if self.call36_tuple(args) > 0:
self.write(', ')
pass
pass
self.write('*')
self.preorder(node[1][1])
self.write(', ')
kwargs = node[2]
if kwargs == 'expr':
kwargs = kwargs[0]
if kwargs == 'dict':
self.call36_dict(kwargs)
else:
self.write('**')
self.preorder(kwargs)
self.write(')')
self.prune()
self.n_call_ex_kw3 = call_ex_kw3
def call_ex_kw4(node):
"""Handle CALL_FUNCTION_EX {1 or 2} but without
BUILD_{MAP,TUPLE}_UNPACK_WITH_CALL"""
self.preorder(node[0])
self.write('(')
args = node[1][0]
if args == 'tuple':
if self.call36_tuple(args) > 0:
self.write(', ')
pass
pass
else:
self.write('*')
self.preorder(args)
self.write(', ')
pass
kwargs = node[2]
if kwargs == 'expr':
kwargs = kwargs[0]
call_function_ex = node[-1]
assert call_function_ex == 'CALL_FUNCTION_EX_KW'
# FIXME: decide if the below test be on kwargs == 'dict'
if (call_function_ex.attr & 1 and
(not isinstance(kwargs, Token) and kwargs != 'attribute')
and not kwargs[0].kind.startswith('kvlist')):
self.call36_dict(kwargs)
else:
self.write('**')
self.preorder(kwargs)
self.write(')')
self.prune()
self.n_call_ex_kw4 = call_ex_kw4
def call36_tuple(node):
"""
A tuple used in a call, these are like normal tuples but they
don't have the enclosing parenthesis.
"""
assert node == 'tuple'
# Note: don't iterate over last element which is a
# BUILD_TUPLE...
flat_elems = flatten_list(node[:-1])
self.indent_more(INDENT_PER_LEVEL)
sep = ''
for elem in flat_elems:
if elem in ('ROT_THREE', 'EXTENDED_ARG'):
continue
assert elem == 'expr'
line_number = self.line_number
value = self.traverse(elem)
if line_number != self.line_number:
sep += '\n' + self.indent + INDENT_PER_LEVEL[:-1]
self.write(sep, value)
sep = ', '
self.indent_less(INDENT_PER_LEVEL)
return len(flat_elems)
self.call36_tuple = call36_tuple
def call36_dict(node):
"""
A dict used in a call_ex_kw2, which are a dictionary items expressed
in a call. This should format to:
a=1, b=2
In other words, no braces, no quotes around keys and ":" becomes
"=".
We will source-code use line breaks to guide us when to break.
"""
p = self.prec
self.prec = 100
self.indent_more(INDENT_PER_LEVEL)
sep = INDENT_PER_LEVEL[:-1]
line_number = self.line_number
if node[0].kind.startswith('kvlist'):
# Python 3.5+ style key/value list in dict
kv_node = node[0]
l = list(kv_node)
i = 0
length = len(l)
# FIXME: Parser-speed improved grammars will have BUILD_MAP
# at the end. So in the future when everything is
# complete, we can do an "assert" instead of "if".
if kv_node[-1].kind.startswith("BUILD_MAP"):
length -= 1
# Respect line breaks from source
while i < length:
self.write(sep)
name = self.traverse(l[i], indent='')
# Strip off beginning and trailing quotes in name
name = name[1:-1]
if i > 0:
line_number = self.indent_if_source_nl(line_number,
self.indent + INDENT_PER_LEVEL[:-1])
line_number = self.line_number
self.write(name, '=')
value = self.traverse(l[i+1], indent=self.indent+(len(name)+2)*' ')
self.write(value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
line_number = self.line_number
i += 2
pass
elif node[-1].kind.startswith('BUILD_CONST_KEY_MAP'):
keys_node = node[-2]
keys = keys_node.attr
# from trepan.api import debug; debug()
assert keys_node == 'LOAD_CONST' and isinstance(keys, tuple)
for i in range(node[-1].attr):
self.write(sep)
self.write(keys[i], '=')
value = self.traverse(node[i], indent='')
self.write(value)
sep = ", "
if line_number != self.line_number:
sep += "\n" + self.indent + INDENT_PER_LEVEL[:-1]
line_number = self.line_number
pass
pass
else:
self.write("**")
try:
self.default(node)
except GenericASTTraversalPruningException:
pass
self.prec = p
self.indent_less(INDENT_PER_LEVEL)
return
self.call36_dict = call36_dict
FSTRING_CONVERSION_MAP = {1: '!s', 2: '!r', 3: '!a'}
def n_except_suite_finalize(node):
if node[1] == 'returns' and self.hide_internal:
# Process node[1] only.
# The code after "returns", e.g. node[3], is dead code.
# Adding it is wrong as it dedents and another
# exception handler "except_stmt" afterwards.
# Note it is also possible that the grammar is wrong here.
# and this should not be "except_stmt".
self.indent_more()
self.preorder(node[1])
self.indent_less()
else:
self.default(node)
self.prune()
self.n_except_suite_finalize = n_except_suite_finalize
def n_formatted_value(node):
if node[0] == 'LOAD_CONST':
self.write(node[0].attr)
self.prune()
else:
self.default(node)
self.n_formatted_value = n_formatted_value
def f_conversion(node):
node.conversion = FSTRING_CONVERSION_MAP.get(node.data[1].attr, '')
def fstring_expr(node):
f_conversion(node)
self.default(node)
self.n_fstring_expr = fstring_expr
def fstring_single(node):
f_conversion(node)
self.default(node)
self.n_fstring_single = fstring_single
# def kwargs_only_36(node):
# keys = node[-1].attr
# num_kwargs = len(keys)
# values = node[:num_kwargs]
# for i, (key, value) in enumerate(zip(keys, values)):
# self.write(key + '=')
# self.preorder(value)
# if i < num_kwargs:
# self.write(',')
# self.prune()
# return
# self.n_kwargs_only_36 = kwargs_only_36
def n_call_kw36(node):
self.template_engine(("%c(", 0), node)
keys = node[-2].attr
num_kwargs = len(keys)
num_posargs = len(node) - (num_kwargs + 2)
n = len(node)
assert n >= len(keys)+1, \
'not enough parameters keyword-tuple values'
sep = ''
line_number = self.line_number
for i in range(1, num_posargs):
self.write(sep)
self.preorder(node[i])
if line_number != self.line_number:
sep = ",\n" + self.indent + " "
else:
sep = ", "
line_number = self.line_number
i = num_posargs
j = 0
# FIXME: adjust output for line breaks?
while i < n-2:
self.write(sep)
self.write(keys[j] + '=')
self.preorder(node[i])
if line_number != self.line_number:
sep = ",\n" + self.indent + " "
else:
sep = ", "
i += 1
j += 1
self.write(')')
self.prune()
return
self.n_call_kw36 = n_call_kw36
def starred(node):
l = len(node)
assert l > 0
pos_args = node[0]
if pos_args == 'expr':
pos_args = pos_args[0]
if pos_args == 'tuple':
build_tuple = pos_args[0]
if build_tuple.kind.startswith('BUILD_TUPLE'):
tuple_len = 0
else:
tuple_len = len(node) - 1
star_start = 1
template = '%C', (0, -1, ', ')
self.template_engine(template, pos_args)
if tuple_len == 0:
self.write("*()")
# That's it
self.prune()
self.write(', ')
else:
star_start = 0
if l > 1:
template = ( '*%C', (star_start, -1, ', *') )
else:
template = ( '*%c', (star_start, 'expr') )
self.template_engine(template, node)
self.prune()
self.n_starred = starred
def return_closure(node):
# Nothing should be output here
self.prune()
return
self.n_return_closure = return_closure
pass # version >= 3.6
pass # version >= 3.4
return

24
uncompyle6/semantics/make_function.py
View File

@@ -506,6 +506,10 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
lambda_index = None
args_node = node[-1]
# Get a list of tree nodes that constitute the values for the "default
# parameters"; these are default values that appear before any *, and are
# not to be confused with keyword parameters which may appear after *.
if isinstance(args_node.attr, tuple):
pos_args, kw_args, annotate_argc = args_node.attr
# FIXME: there is probably a better way to classify this.
@@ -516,13 +520,23 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
lc_index = -3
pass
if (self.version <= 3.3 and len(node) > 2 and
if (3.1 <= self.version <= 3.3 and len(node) > 2 and
node[lambda_index] != 'LOAD_LAMBDA' and
(have_kwargs or node[lc_index].kind != 'load_closure')):
# Find the index in "node" where the first default
# parameter value is located. Note this is in contrast to
# key-word arguments, pairs of (name, value), which appear after "*".
# "default_values_start" is this location.
default_values_start = 0
if node[0] == 'no_kwargs':
default_values_start += 1
# args are after kwargs; kwargs are bundled as one node
defparams = node[1:args_node.attr[0]+1]
if node[default_values_start] == 'kwargs':
default_values_start += 1
defparams = node[default_values_start:default_values_start+args_node.attr[0]]
else:
# args are before kwargs; kwags as bundled as one node
# args are first, before kwargs. Or there simply are no kwargs.
defparams = node[:args_node.attr[0]]
pass
else:
@@ -581,7 +595,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
paramnames = list(scanner_code.co_varnames[:argc])
# defaults are for last n parameters, thus reverse
if not 3.0 <= self.version <= 3.1 or self.version >= 3.6:
if not 3.0 == self.version or self.version >= 3.6:
paramnames.reverse(); defparams.reverse()
try:
@@ -611,7 +625,7 @@ def make_function3(self, node, is_lambda, nested=1, codeNode=None):
else:
params = paramnames
if not 3.0 <= self.version <= 3.1 or self.version >= 3.6:
if not 3.0 == self.version or self.version >= 3.6:
params.reverse() # back to correct order
if code_has_star_arg(code):

257
uncompyle6/semantics/pysource.py
View File

@@ -135,7 +135,7 @@ from spark_parser import GenericASTTraversal, DEFAULT_DEBUG as PARSER_DEFAULT_DE
from uncompyle6.scanner import Code, get_scanner
import uncompyle6.parser as python_parser
from uncompyle6.semantics.make_function import (
make_function2, make_function3, make_function3_annotate,
make_function2, make_function3
)
from uncompyle6.semantics.parser_error import ParserError
from uncompyle6.semantics.check_ast import checker
@@ -147,7 +147,7 @@ from uncompyle6.scanners.tok import Token
from uncompyle6.semantics.consts import (
LINE_LENGTH, RETURN_LOCALS, NONE, RETURN_NONE, PASS,
ASSIGN_DOC_STRING, NAME_MODULE, TAB,
INDENT_PER_LEVEL, TABLE_R, TABLE_DIRECT, MAP_DIRECT,
INDENT_PER_LEVEL, TABLE_R, MAP_DIRECT,
MAP, PRECEDENCE, ASSIGN_TUPLE_PARAM, escape, minint)
@@ -256,223 +256,6 @@ class SourceWalker(GenericASTTraversal, object):
self.write("\n" + self.indent + INDENT_PER_LEVEL[:-1])
return self.line_number
def customize_for_version(self, is_pypy, version):
if is_pypy:
########################
# PyPy changes
#######################
TABLE_DIRECT.update({
'assert_pypy': ( '%|assert %c\n' , 1 ),
'assert2_pypy': ( '%|assert %c, %c\n' , 1, 4 ),
'try_except_pypy': ( '%|try:\n%+%c%-%c\n\n', 1, 2 ),
'tryfinallystmt_pypy': ( '%|try:\n%+%c%-%|finally:\n%+%c%-\n\n', 1, 3 ),
'assign3_pypy': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 4, 3, 0, 1, 2 ),
'assign2_pypy': ( '%|%c, %c = %c, %c\n', 3, 2, 0, 1),
})
else:
########################
# Without PyPy
#######################
TABLE_DIRECT.update({
'assert': ( '%|assert %c\n' , 0 ),
'assert2': ( '%|assert %c, %c\n' , 0, 3 ),
'try_except': ( '%|try:\n%+%c%-%c\n\n', 1, 3 ),
'assign2': ( '%|%c, %c = %c, %c\n', 3, 4, 0, 1 ),
'assign3': ( '%|%c, %c, %c = %c, %c, %c\n', 5, 6, 7, 0, 1, 2 ),
})
if version < 3.0:
TABLE_R.update({
'STORE_SLICE+0': ( '%c[:]', 0 ),
'STORE_SLICE+1': ( '%c[%p:]', 0, (1, 100) ),
'STORE_SLICE+2': ( '%c[:%p]', 0, (1, 100) ),
'STORE_SLICE+3': ( '%c[%p:%p]', 0, (1, 100), (2, 100) ),
'DELETE_SLICE+0': ( '%|del %c[:]\n', 0 ),
'DELETE_SLICE+1': ( '%|del %c[%c:]\n', 0, 1 ),
'DELETE_SLICE+2': ( '%|del %c[:%c]\n', 0, 1 ),
'DELETE_SLICE+3': ( '%|del %c[%c:%c]\n', 0, 1, 2 ),
})
TABLE_DIRECT.update({
'raise_stmt2': ( '%|raise %c, %c\n', 0, 1),
})
else:
TABLE_DIRECT.update({
# Gotta love Python for its futzing around with syntax like this
'raise_stmt2': ( '%|raise %c from %c\n', 0, 1),
})
if version >= 3.2:
TABLE_DIRECT.update({
'del_deref_stmt': ( '%|del %c\n', 0),
'DELETE_DEREF': ( '%{pattr}', 0 ),
})
if version <= 2.4:
TABLE_DIRECT.update({
'importmultiple': ( '%|import %c%c\n', 2, 3),
'import_cont' : ( ', %c', 2),
'tryfinallystmt': ( '%|try:\n%+%c%-%|finally:\n%+%c%-',
(1, 'suite_stmts_opt') ,
(5, 'suite_stmts_opt') )
})
if version == 2.3:
TABLE_DIRECT.update({
'if1_stmt': ( '%|if 1\n%+%c%-', 5 )
})
global NAME_MODULE
NAME_MODULE = AST('stmt',
[ AST('assign',
[ AST('expr',
[Token('LOAD_GLOBAL', pattr='__name__',
offset=0, has_arg=True)]),
AST('store',
[ Token('STORE_NAME', pattr='__module__',
offset=3, has_arg=True)])
])])
pass
if version <= 2.3:
if version <= 2.1:
TABLE_DIRECT.update({
'importmultiple': ( '%c', 2 ),
# FIXME: not quite right. We have indiividual imports
# when there is in fact one: "import a, b, ..."
'imports_cont': ( '%C%,', (1, 100, '\n') ),
})
pass
pass
pass
elif version >= 2.5:
########################
# Import style for 2.5+
########################
TABLE_DIRECT.update({
'importmultiple': ( '%|import %c%c\n', 2, 3 ),
'import_cont' : ( ', %c', 2 ),
# With/as is allowed as "from future" thing in 2.5
# Note: It is safe to put the variables after "as" in parenthesis,
# and sometimes it is needed.
'withstmt': ( '%|with %c:\n%+%c%-', 0, 3),
'withasstmt': ( '%|with %c as (%c):\n%+%c%-', 0, 2, 3),
})
# In 2.5+ "except" handlers and the "finally" can appear in one
# "try" statement. So the below has the effect of combining the
# "tryfinally" with statement with the "try_except" statement
def tryfinallystmt(node):
if len(node[1][0]) == 1 and node[1][0][0] == 'stmt':
if node[1][0][0][0] == 'try_except':
node[1][0][0][0].kind = 'tf_try_except'
if node[1][0][0][0] == 'tryelsestmt':
node[1][0][0][0].kind = 'tf_tryelsestmt'
self.default(node)
self.n_tryfinallystmt = tryfinallystmt
########################################
# Python 2.6+
# except <condition> as <var>
# vs. older:
# except <condition> , <var>
#
# For 2.6 we use the older syntax which
# matches how we parse this in bytecode
########################################
if version > 2.6:
TABLE_DIRECT.update({
'except_cond2': ( '%|except %c as %c:\n', 1, 5 ),
})
else:
TABLE_DIRECT.update({
'except_cond3': ( '%|except %c, %c:\n', 1, 6 ),
'testtrue_then': ( 'not %p', (0, 22) ),
})
if 2.4 <= version <= 2.6:
TABLE_DIRECT.update({
'comp_for': ( ' for %c in %c', 3, 1 ),
})
else:
TABLE_DIRECT.update({
'comp_for': ( ' for %c in %c%c', 2, 0, 3 ),
})
if version >= 3.0:
TABLE_DIRECT.update({
'function_def_annotate': ( '\n\n%|def %c%c\n', -1, 0),
'store_locals': ( '%|# inspect.currentframe().f_locals = __locals__\n', ),
})
def n_mkfunc_annotate(node):
if self.version >= 3.3 or node[-2] in ('kwargs', 'no_kwargs'):
# LOAD_CONST code object ..
# LOAD_CONST 'x0' if >= 3.3
# EXTENDED_ARG
# MAKE_FUNCTION ..
code = node[-4]
elif node[-3] == 'expr':
code = node[-3][0]
else:
# LOAD_CONST code object ..
# MAKE_FUNCTION ..
code = node[-3]
self.indent_more()
for annotate_last in range(len(node)-1, -1, -1):
if node[annotate_last] == 'annotate_tuple':
break
# FIXME: the real situation is that when derived from
# function_def_annotate we the name has been filled in.
# But when derived from funcdefdeco it hasn't Would like a better
# way to distinquish.
if self.f.getvalue()[-4:] == 'def ':
self.write(code.attr.co_name)
# FIXME: handle and pass full annotate args
make_function3_annotate(self, node, is_lambda=False,
codeNode=code, annotate_last=annotate_last)
if len(self.param_stack) > 1:
self.write('\n\n')
else:
self.write('\n\n\n')
self.indent_less()
self.prune() # stop recursing
self.n_mkfunc_annotate = n_mkfunc_annotate
if version >= 3.4:
########################
# Python 3.4+ Additions
#######################
TABLE_DIRECT.update({
'LOAD_CLASSDEREF': ( '%{pattr}', ),
})
########################
# Python 3.5+ Additions
#######################
if version >= 3.5:
TABLE_DIRECT.update({
'await_expr': ( 'await %c', 0),
'await_stmt': ( '%|%c\n', 0),
'async_for_stmt': (
'%|async for %c in %c:\n%+%c%-\n\n', 9, 1, 25 ),
'async_forelse_stmt': (
'%|async for %c in %c:\n%+%c%-%|else:\n%+%c%-\n\n', 9, 1, 25, 28 ),
'async_with_stmt': (
'%|async with %c:\n%+%c%-', 0, 7),
'async_with_as_stmt': (
'%|async with %c as %c:\n%+%c%-', 0, 6, 7),
'unmap_dict': ( '{**%C}', (0, -1, ', **') ),
# 'unmapexpr': ( '{**%c}', 0), # done by n_unmapexpr
})
pass # version >= 3.4
pass # version >= 3.0
return
f = property(lambda s: s.params['f'],
lambda s, x: s.params.__setitem__('f', x),
lambda s: s.params.__delitem__('f'),
@@ -624,26 +407,14 @@ class SourceWalker(GenericASTTraversal, object):
self.prune() # stop recursing
def n_yield(self, node):
self.write('yield')
if node != AST('yield', [NONE, Token('YIELD_VALUE')]):
self.write(' ')
self.preorder(node[0])
self.template_engine(( 'yield %c', 0), node)
elif self.version <= 2.4:
# Early versions of Python don't allow a plain "yield"
self.write(' None')
self.prune() # stop recursing
# In Python 3.3+ only
def n_yield_from(self, node):
self.write('yield from')
self.write(' ')
if 3.3 <= self.version <= 3.4:
self.preorder(node[0][0][0][0])
elif self.version >= 3.5:
self.preorder(node[0])
self.write('yield None')
else:
assert False, "dunno about this python version"
self.write('yield')
self.prune() # stop recursing
def n_build_slice3(self, node):
@@ -1622,9 +1393,15 @@ class SourceWalker(GenericASTTraversal, object):
def print_super_classes3(self, node):
n = len(node) - 1
if node.kind != 'expr':
assert node[n].kind.startswith('CALL_FUNCTION')
if node == 'kwarg':
self.write('(')
self.template_engine(('%[0]{pattr}=%c', 1), node)
self.write(')')
return
kwargs = None
assert node[n].kind.startswith('CALL_FUNCTION')
if node[n].kind.startswith('CALL_FUNCTION_KW'):
# 3.6+ starts does this
kwargs = node[n-1].attr
@@ -1632,12 +1409,13 @@ class SourceWalker(GenericASTTraversal, object):
i = n - (len(kwargs)+1)
j = 1 + n - node[n].attr
else:
for i in range(n-2, 0, -1):
if not node[i].kind in ['expr', 'LOAD_CLASSNAME']:
start = n-2
for i in range(start, 0, -1):
if not node[i].kind in ['expr', 'call', 'LOAD_CLASSNAME']:
break
pass
if i == n-2:
if i == start:
return
i += 2
@@ -1905,6 +1683,7 @@ class SourceWalker(GenericASTTraversal, object):
else:
self.write('('); endchar = ')'
pass
elif lastnodetype.startswith('BUILD_SET'):
self.write('{'); endchar = '}'
elif lastnodetype.startswith('BUILD_MAP_UNPACK'):