Merge branch 'master' into python-2.4

This commit is contained in:
rocky
2017-10-12 07:05:34 -04:00
25 changed files with 349 additions and 22 deletions

168
ChangeLog
View File

@@ -1,7 +1,65 @@
2017-10-11 rocky <rb@dustyfeet.com>
* admin-tools/check-newer-versions.sh,
admin-tools/check-older-versions.sh,
admin-tools/how-to-make-a-release.txt,
admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh,
admin-tools/pyenv-newer-versions, admin-tools/pyenv-older-versions,
uncompyle6/parsers/parse37.py, uncompyle6/scanners/scanner37.py,
uncompyle6/version.py: More administrivia
2017-10-11 rocky <rb@dustyfeet.com>
* admin-tools/README.md, admin-tools/check-newer-versions.sh,
admin-tools/check-older-versions.sh,
admin-tools/how-to-make-a-release.txt,
admin-tools/make-dist-newer.sh, admin-tools/make-dist-older.sh,
admin-tools/pyenv-newer-versions, admin-tools/pyenv-older-versions,
admin-tools/setup-master.sh, admin-tools/setup-python-2.4.sh: Some
admin tools I use
2017-10-11 rocky <rb@dustyfeet.com>
* uncompyle6/parser.py: Remove creaping Python 2.6ism
2017-10-10 rocky <rb@dustyfeet.com>
* uncompyle6/parsers/parse24.py, uncompyle6/scanners/scanner3.py:
Misc bugs
* pytest/test_grammar.py: Sync with master
2017-10-10 rocky <rb@dustyfeet.com>
* pytest/test_grammar.py, uncompyle6/parsers/parse26.py,
uncompyle6/parsers/parse27.py, uncompyle6/parsers/parse34.py,
uncompyle6/parsers/parse35.py, uncompyle6/parsers/parse36.py,
uncompyle6/parsers/parse37.py: Sync with master
2017-10-10 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py: Sync with master
2017-10-10 rocky <rb@dustyfeet.com>
* NEWS, __pkginfo__.py, pytest/test_grammar.py,
uncompyle6/parser.py, uncompyle6/parsers/parse15.py,
uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse21.py,
uncompyle6/parsers/parse22.py, uncompyle6/parsers/parse23.py,
uncompyle6/parsers/parse24.py, uncompyle6/parsers/parse25.py,
uncompyle6/parsers/parse26.py, uncompyle6/parsers/parse27.py,
uncompyle6/parsers/parse3.py, uncompyle6/parsers/parse32.py,
uncompyle6/parsers/parse34.py, uncompyle6/parsers/parse35.py,
uncompyle6/parsers/parse36.py, uncompyle6/parsers/parse37.py,
uncompyle6/scanners/scanner22.py, uncompyle6/scanners/scanner26.py,
uncompyle6/scanners/scanner27.py, uncompyle6/scanners/scanner3.py,
uncompyle6/scanners/scanner36.py, uncompyle6/scanners/tok.py,
uncompyle6/semantics/check_ast.py,
uncompyle6/semantics/make_function.py,
uncompyle6/semantics/pysource.py, uncompyle6/verify.py,
uncompyle6/version.py: Sync with master
2017-10-10 rocky <rb@dustyfeet.com>
* : Merge commit '1d7a3c6444eab5a02d899f789f2a57cfdcbc5a84' into
python-2.4
2017-10-05 rocky <rb@dustyfeet.com>
@@ -31,10 +89,34 @@
* uncompyle6/parsers/parse2.py, uncompyle6/parsers/parse3.py: Sync
with master
2017-09-30 rocky <rb@dustyfeet.com>
* uncompyle6/parser.py, uncompyle6/scanners/scanner2.py,
uncompyle6/scanners/scanner3.py: Document hacky customize arg count
better.
2017-09-26 rocky <rb@dustyfeet.com>
* README.rst: Word hacking
2017-09-26 rocky <rb@dustyfeet.com>
* ChangeLog, NEWS: Get ready for release 2.12.0
2017-09-26 rocky <rb@dustyfeet.com>
* uncompyle6/parsers/parse3.py: Annotation field can be unicode... When deparsing Python 3.x from Python 2.
2017-09-26 rocky <rb@dustyfeet.com>
* uncompyle6/parsers/parse3.py: No unicode in Python3. but we need it in Python2. The bug was probably introduced as a
result of recent Python code type unteroperability canonicalization
2017-09-26 rocky <rb@dustyfeet.com>
* uncompyle6/parsers/parse3.py: Pyton 3.1 Annotation args can be
unicode?
2017-09-25 rocky <rb@dustyfeet.com>
* __pkginfo__.py: Require xdis 3.6.0 or greater
@@ -44,38 +126,120 @@
* : commit 0654aed6c823d0bb20abdc866481ca5950db72f7 Author: rocky
<rb@dustyfeet.com> Date: Thu Sep 21 11:29:17 2017 -0400
2017-09-25 rocky <rb@dustyfeet.com>
* : Adjust for xdis opcode JUMP_OPS. release 2.12.0
2017-09-21 rocky <rb@dustyfeet.com>
* pytest/test_pysource.py: Python 3 compatibility
2017-09-21 rocky <rb@dustyfeet.com>
* pytest/test_pysource.py, uncompyle6/semantics/consts.py,
uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py:
Unit test for format-specifiers
2017-09-21 rocky <rb@dustyfeet.com>
* pytest/test_pysource.py, uncompyle6/semantics/consts.py,
uncompyle6/semantics/fragments.py, uncompyle6/semantics/pysource.py:
Unit test for format-specifiers And in the process we catch some small bugs
2017-09-20 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py,
uncompyle6/semantics/pysource.py: Tidy pysource and fragments
2017-09-20 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py,
uncompyle6/semantics/pysource.py: Tidy pysource and fragments a
little more
2017-09-20 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/consts.py: Tidy/regularize table entry
formatting
2017-09-20 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/consts.py: Tidy/regularize table entry
formatting
2017-09-20 rocky <rb@dustyfeet.com>
* test/test_pythonlib.py, uncompyle6/semantics/pysource.py: Small
fixes test_pyenvlib.py: it is sys.exit(), not exit() pysource.py:
reinstate nod type of async_func_call
2017-09-20 rocky <rb@dustyfeet.com>
* test/test_pythonlib.py, uncompyle6/semantics/pysource.py: small
fixes... test_pythonlib.py: it is sys.exit not exit pysource.py: restore node
type on async_call function
2017-09-20 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/consts.py, uncompyle6/semantics/pysource.py:
More small doc changes
2017-09-20 rocky <rb@dustyfeet.com>
* pytest/test_pysource.py, uncompyle6/semantics/pysource.py: Start
pysource unit test
2017-09-20 rocky <rb@dustyfeet.com>
* pytest/test_pysource.py, uncompyle6/semantics/pysource.py: Update
Table-driven info... Start a pysource unit test.
2017-09-17 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py,
uncompyle6/semantics/pysource.py: emgine -> template_engine
2017-09-17 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py,
uncompyle6/semantics/pysource.py: engine -> template_engine
2017-09-13 rocky <rb@dustyfeet.com>
* test/Makefile: Need weak-verification on 3.4 for now
2017-09-10 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py: Revert one of the changes
pending a better fix
2017-09-10 rocky <rb@dustyfeet.com>
* uncompyle6/semantics/fragments.py,
uncompyle6/semantics/pysource.py: More semantic action cleanup
2017-09-10 rocky <rb@dustyfeet.com>
* uncompyle6/scanners/scanner3.py, uncompyle6/scanners/tok.py: Match
Python 3.4's terms a little names better
2017-09-09 rocky <rb@dustyfeet.com>
* uncompyle6/scanners/tok.py: Revert last revert
2017-09-09 rocky <rb@dustyfeet.com>
* uncompyle6/scanners/tok.py: Revert last change
2017-09-09 rocky <rb@dustyfeet.com>
* uncompyle6/scanners/tok.py: New-style Python classes only, please.
2017-08-31 rocky <rb@dustyfeet.com>
* uncompyle6/scanner.py, uncompyle6/scanners/scanner37.py: Skeletal
support for Python 3.7 Largely failing though.
2017-08-31 rocky <rb@dustyfeet.com>
* : commit 356ea6c7705a557cb3e725d1aca8589dd62b5cdf Author: rocky

View File

@@ -3,8 +3,9 @@
## The difficulty of the problem
There is no Python decompiler yet, that I know about that will
decompyle everything. This one probably does the
best job of *any* Python decompiler. But it is a constant work in progress: Python keeps changing, and so does its code generation.
decompyle everything. This one probably does the best job of *any*
Python decompiler. But it is a constant work in progress: Python keeps
changing, and so does its code generation.
I have found bugs in *every* Python decompiler I have tried. Even
those where authors/maintainers claim that they have used it on
@@ -14,6 +15,55 @@ but that the program is *semantically* not equivalent.
So it is likely you'll find a mistranslation in decompiling.
## Is it really a bug?
If the code emitted is semantically equivalent, then this isn't a bug.
For example the code might be
```
if a:
if b:
x = 1
```
and we might produce:
```
if a and b:
x = 1
```
These are equivalent. Sometimes
```
else:
if ...
```
may out as `elif`.
As mentioned in the README. It is possible that Python changes what
you write to be more efficient. For example, for:
```
if True:
x = 5
```
Python will generate code like:
```
x = 5
```
So just because the text isn't the same, does not
necessarily mean there's a bug.
## What to send (minimum requirements)
The basic requirement is pretty simple:
@@ -21,6 +71,12 @@ The basic requirement is pretty simple:
* Python bytecode
* Python source text
Please don't put files on download services that one has to register
for. If you can't attach it to the issue, or create a github gist,
then the code you are sending is too large.
Please also try to narrow the bug. See below.
## What to send (additional helpful information)
Some kind folks also give the invocation they used and the output

View File

@@ -11,7 +11,7 @@ RM ?= rm
LINT = flake8
#EXTRA_DIST=ipython/ipy_trepan.py trepan
PHONY=all check clean pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
PHONY=all check clean distcheck pytest check-long dist distclean lint flake8 test rmChangeLog clean_pyc
TEST_TYPES=check-long check-short check-2.7 check-3.4
@@ -60,8 +60,12 @@ clean: clean_pyc
(cd test && $(MAKE) clean)
#: Create source (tarball) and wheel distribution
dist:
$(PYTHON) ./setup.py sdist bdist_egg
dist: distcheck
$(PYTHON) ./setup.py sdist bdist_wheel
# perform some checks on the package via setup.py
distcheck:
$(PYTHON) ./setup.py check
#: Remove .pyc files
clean_pyc:
@@ -89,7 +93,7 @@ bdist_egg:
#: Create binary wheel distribution
bdist_wheel:
wheel:
$(PYTHON) ./setup.py bdist_wheel

12
NEWS
View File

@@ -1,3 +1,14 @@
<<<<<<< HEAD
=======
uncompyle6 2.13.2 2017-10-12
- Re-release using a more automated approach
uncompyle6 2.13.1 2017-10-11
- Re-release because Python 2.4 source uploaded rather than 2.6-3.6
>>>>>>> master
uncompyle6 2.13.0 2017-10-10
- Fixes in deparsing lambda expressions
@@ -11,6 +22,7 @@ uncompyle6 2.12.0 2017-09-26
- Small semantic table cleanups
- Python 3.4's terms a little names better
- Slightly more Python 3.7, but still failing a lot
- Cross Python 2/3 compatibility with annotation arguments
uncompyle6 2.11.5 2017-08-31

View File

@@ -5,8 +5,16 @@ PACKAGE=uncompyle6
function finish {
cd $owd
}
<<<<<<< HEAD
cd $(dirname ${BASH_SOURCE[0]})
if ! source ./pyenv-older-versions ; then
=======
owd=$(pwd)
trap finish EXIT
cd $(dirname ${BASH_SOURCE[0]})
if ! source ./pyenv-newer-versions ; then
>>>>>>> master
exit $?
fi
if ! source ./setup-master.sh ; then
@@ -18,7 +26,17 @@ source $PACKAGE/version.py
echo $VERSION
for pyversion in $PYVERSIONS; do
<<<<<<< HEAD
# Pick out first two numbers
=======
if ! pyenv local $pyversion ; then
exit $?
fi
# pip bdist_egg create too-general wheels. So
# we narrow that by moving the generated wheel.
# Pick out first two number of version, e.g. 3.5.1 -> 35
>>>>>>> master
first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
rm -fr build
python setup.py bdist_egg bdist_wheel

View File

@@ -16,8 +16,12 @@ if ! source ./setup-python-2.4.sh ; then
exit $?
fi
<<<<<<< HEAD
cd ..
source $PACKAGE/version.py
=======
source ../$PACKAGE/version.py
>>>>>>> master
echo $VERSION
for pyversion in $PYVERSIONS; do
@@ -34,6 +38,10 @@ done
# the tarball from master.
tarball=dist/uncompyle6-$VERSION-tar.gz
<<<<<<< HEAD
if [[ -f $tarball ]]; then
=======
if -f $tarball; then
>>>>>>> master
rm -v dist/uncompyle6-$VERSION-tar.gz
fi

View File

@@ -1,5 +1,9 @@
# -*- shell-script -*-
<<<<<<< HEAD
if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
=======
if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
>>>>>>> master
echo "This script should be *sourced* rather than run directly through bash"
exit 1
fi

View File

@@ -1,6 +1,12 @@
#!/bin/bash
PYTHON_VERSION=3.6.3
# FIXME put some of the below in a common routine
function finish {
cd $owd
}
export PATH=$HOME/.pyenv/bin/pyenv:$PATH
owd=$(pwd)
bs=${BASH_SOURCE[0]}
if [[ $0 == $bs ]] ; then

View File

@@ -53,7 +53,7 @@ def test_tables():
# One arg - should be int or tuple of int
if typ == 'c':
assert isinstance(entry[arg], int), (
"%s[%s][%d] type %s is '%s' should be an int but is %s. "
"%s[%s][%d] kind %s is '%s' should be an int but is %s. "
"Full entry: %s" %
(name, k, arg, typ, entry[arg], type(entry[arg]), entry)
)

View File

@@ -47,7 +47,7 @@ check-3.5: check-bytecode
#: Run working tests from Python 3.6
check-3.6: check-bytecode
$(PYTHON) test_pythonlib.py --bytecode-3.6 --verify $(COMPILE)
$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
#: Check deparsing only, but from a different Python version
check-disasm:

Binary file not shown.

Binary file not shown.

View File

@@ -3,3 +3,17 @@
f = lambda x: 1 if x<2 else 3
f(5)
<<<<<<< HEAD
=======
# If that wasn't enough ...
# Python will create dead code
# in the below. So we must make sure
# not to include the else expression
g = lambda: 1 if True else 3
g()
h = lambda: 1 if False else 3
h()
>>>>>>> master

View File

@@ -1,12 +1,12 @@
#!/usr/bin/env python
# Mode: -*- python -*-
#
# Copyright (c) 2015-2016 by Rocky Bernstein
# Copyright (c) 2015-2017 by Rocky Bernstein
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
#
import sys, os, getopt, time
program, ext = os.path.splitext(os.path.basename(__file__))
program = 'uncompyle6'
__doc__ = """
Usage:

View File

@@ -96,10 +96,14 @@ class PythonParser(GenericASTBuilder):
def fix(c):
s = str(c)
last_token_pos = s.find('_')
<<<<<<< HEAD
if last_token_pos == -1:
return s
else:
return s[:last_token_pos]
=======
return s if last_token_pos == -1 else s[:last_token_pos]
>>>>>>> master
prefix = ''
if parent and tokens:

View File

@@ -16,7 +16,7 @@ class AST(spark_AST):
return self.__repr1__('', None)
def __repr1__(self, indent, sibNum=None):
rv = str(self.type)
rv = str(self.kind)
if sibNum is not None:
rv = "%2d. %s" % (sibNum, rv)
enumerate_children = False

View File

@@ -155,8 +155,13 @@ class Python3Parser(PythonParser):
# of missing "else" clauses. Therefore we include grammar
# rules with and without ELSE.
ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite opt_come_from_except
ifelsestmt ::= testexpr c_stmts_opt jump_forward_else else_suite _come_from
ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD
else_suite opt_come_from_except
ifelsestmt ::= testexpr c_stmts_opt jump_forward_else
else_suite _come_from
# ifelsestmt ::= testexpr c_stmts_opt jump_forward_else
# passstmt _come_from
ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
@@ -252,8 +257,14 @@ class Python3Parser(PythonParser):
POP_BLOCK LOAD_CONST COME_FROM_WITH
WITH_CLEANUP END_FINALLY
## FIXME: Right now we have erroneous jump targets
## This below is probably not correct when the COME_FROM is put in the right place
and ::= expr jmp_false expr COME_FROM
or ::= expr jmp_true expr COME_FROM
# # something like the below is needed when the jump targets are fixed
## or ::= expr JUMP_IF_TRUE_OR_POP COME_FROM expr
## and ::= expr JUMP_IF_FALSE_OR_POP COME_FROM expr
'''
def p_misc3(self, args):

View File

@@ -785,6 +785,10 @@ class Scanner3(Scanner):
if ((code[prev_op[target]] in self.pop_jump_if_pop) and
(target > offset) and prev_op[target] != offset):
# FIXME: this is not accurate The commented out below
# is what it should be. However grammar rules right now
# assume the incorrect offsets.
# self.fixed_jumps[offset] = target
self.fixed_jumps[offset] = prev_op[target]
self.structs.append({'type': 'and/or',
'start': start,

View File

@@ -55,10 +55,15 @@ class Token:
return self.format(line_prefix='')
def format(self, line_prefix=''):
<<<<<<< HEAD
if self.linestart:
prefix = '\n%s%4d ' % (line_prefix, self.linestart)
else:
prefix = ' ' * (6 + len(line_prefix))
=======
prefix = ('\n%s%4d ' % (line_prefix, self.linestart)
if self.linestart else (' ' * (6 + len(line_prefix))))
>>>>>>> master
offset_opname = '%6s %-17s' % (self.offset, self.kind)
if not self.has_arg:
return "%s%s" % (prefix, offset_opname)

View File

@@ -173,7 +173,11 @@ TABLE_DIRECT = {
'ret_cond': ( '%p if %p else %p', (2, 27), (0, 27), (-1, 27) ),
'conditionalnot': ( '%p if not %p else %p', (2, 27), (0, 22), (4, 27) ),
'ret_cond_not': ( '%p if not %p else %p', (2, 27), (0, 22), (-1, 27) ),
<<<<<<< HEAD
'conditional_lambda': ( '(%c if %c else %c)', 2, 0, 3),
=======
'conditional_lambda': ( '%c if %c else %c', 2, 0, 4),
>>>>>>> master
'compare': ( '%p %[-1]{pattr.replace("-", " ")} %p', (0, 19), (1, 19) ),
'cmp_list': ( '%p %p', (0, 29), (1, 30)),

View File

@@ -451,6 +451,19 @@ def make_function3(self, node, isLambda, nested=1, codeNode=None):
# MAKE_FUNCTION_... or MAKE_CLOSURE_...
assert node[-1].kind.startswith('MAKE_')
<<<<<<< HEAD
=======
# Python 3.3+ adds a qualified name at TOS (-1)
# moving down the LOAD_LAMBDA instruction
if 3.0 <= self.version <= 3.2:
lambda_index = -2
elif 3.03 <= self.version:
lambda_index = -3
else:
lambda_index = None
>>>>>>> master
args_node = node[-1]
if isinstance(args_node.attr, tuple):

View File

@@ -143,7 +143,7 @@ else:
def is_docstring(node):
try:
return (node[0][0].type == 'assign' and
return (node[0][0].kind == 'assign' and
node[0][0][1][0].pattr == '__doc__')
except:
return False
@@ -419,13 +419,13 @@ class SourceWalker(GenericASTTraversal, object):
})
def n_async_call_function(node):
self.f.write('async ')
node.type == 'call_function'
node.kind == 'call_function'
p = self.prec
self.prec = 80
self.template_engine(('%c(%P)', 0,
(1, -4, ', ', 100)), node)
self.prec = p
node.type == 'async_call_function'
node.kind == 'async_call_function'
self.prune()
self.n_async_call_function = n_async_call_function
self.n_build_list_unpack = self.n_build_list
@@ -2235,7 +2235,7 @@ def deparse_code(version, co, out=sys.stdout, showasm=None, showast=False,
debug_parser = dict(PARSER_DEFAULT_DEBUG)
if showgrammar:
debug_parser['reduce'] = showgrammar
debug_parser['errorstack'] = True
debug_parser['errorstack'] = 'full'
# Build AST from disassembly.
linestarts = dict(scanner.opc.findlinestarts(co))

View File

@@ -1,6 +1,6 @@
#
# (C) Copyright 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# (C) Copyright 2015-2016 by Rocky Bernstein
# (C) Copyright 2015-2017 by Rocky Bernstein
#
"""
byte-code verification

View File

@@ -1,3 +1,3 @@
# This file is suitable for sourcing inside bash as
# well as importing into Python
VERSION='2.13.1'
VERSION='2.13.2'