You've already forked python-uncompyle6
mirror of
https://github.com/rocky/python-uncompyle6.git
synced 2025-08-04 01:09:52 +08:00
Compare commits
193 Commits
release-2.
...
release-py
Author | SHA1 | Date | |
---|---|---|---|
|
6d6a73eea7 | ||
|
1e3ea60055 | ||
|
e4a7641927 | ||
|
b24b46d48c | ||
|
2fbbc728b1 | ||
|
0a6c8ba909 | ||
|
d3904527e6 | ||
|
a65d7dce5b | ||
|
718a0a5d34 | ||
|
b043f6bafc | ||
|
aa207a3c77 | ||
|
747212c62c | ||
|
493e4b14a1 | ||
|
9491c67779 | ||
|
8ef5e5d12b | ||
|
222986640e | ||
|
f9d47abb2b | ||
|
31ed869a6f | ||
|
ea9e3ab3f5 | ||
|
19d2569515 | ||
|
770e988ff8 | ||
|
0fa0641974 | ||
|
9348411056 | ||
|
e71dd010d7 | ||
|
dadd1c5c45 | ||
|
99af1c9ffe | ||
|
3dc766d0a9 | ||
|
357005c814 | ||
|
41d63a0261 | ||
|
1cb2cd7a82 | ||
|
9ec312ba5e | ||
|
597d51951e | ||
|
cc2321f49e | ||
|
476a1c8ab5 | ||
|
545a46dffa | ||
|
8333e4ae93 | ||
|
e9057f378a | ||
|
36b75abd90 | ||
|
1528537ca4 | ||
|
6b8ae29267 | ||
|
33ec66a82f | ||
|
b0493d1984 | ||
|
7f37c60c42 | ||
|
e2fd308928 | ||
|
6d7cec002a | ||
|
9c49b5d54b | ||
|
8dc23e2cdc | ||
|
a01b8be054 | ||
|
c13e23cdae | ||
|
114fe11e66 | ||
|
b131c20e99 | ||
|
5db1178b3e | ||
|
fab4ebb768 | ||
|
89429339fa | ||
|
7ece296f76 | ||
|
5035d5433b | ||
|
78a5b620a7 | ||
|
e851c0d46a | ||
|
a760188724 | ||
|
ad345ef94a | ||
|
d050dd3adb | ||
|
9392103998 | ||
|
707770049f | ||
|
ec0669367f | ||
|
3f40c16587 | ||
|
66518baed0 | ||
|
21023fea74 | ||
|
66741d16ba | ||
|
e02ebef45d | ||
|
99fce6dfd7 | ||
|
7b8c5e091c | ||
|
77caf515ea | ||
|
e4c0d56947 | ||
|
4827b1e994 | ||
|
2b46e71264 | ||
|
6ed129bd7a | ||
|
c4fde6b53e | ||
|
a7d93e88b4 | ||
|
84c2932bc5 | ||
|
874b3c9d31 | ||
|
f6a997befc | ||
|
9891494142 | ||
|
f8544dfbbe | ||
|
136f42a610 | ||
|
c43e734f37 | ||
|
b00651d428 | ||
|
2327f0fdfa | ||
|
0afcd31bd5 | ||
|
6f097ff1ca | ||
|
8eb1a16f5b | ||
|
ed9fb64e72 | ||
|
d002c667ae | ||
|
da8dccbaca | ||
|
e56743cc14 | ||
|
39814fab8b | ||
|
970774ab95 | ||
|
723fa5dfed | ||
|
4d4e59c40b | ||
|
a92e6c9688 | ||
|
6c546fe6e1 | ||
|
37272ae827 | ||
|
9b1dd0f26c | ||
|
0ff0c97a95 | ||
|
3e988be075 | ||
|
eb64a03dfa | ||
|
9aa4e2b9ae | ||
|
c147514e9e | ||
|
7f2bee46b7 | ||
|
813229ac45 | ||
|
f1a947f106 | ||
|
2f51067a9d | ||
|
c8a4dcf72b | ||
|
012ff91cfb | ||
|
e3f4beeb74 | ||
|
7d58dcf6dd | ||
|
bfff1b4e9f | ||
|
e6761e13bb | ||
|
e690ddd50a | ||
|
c7c0a98982 | ||
|
eebec48308 | ||
|
45b7c1948c | ||
|
e2fb7ca3d2 | ||
|
da50394841 | ||
|
b3bda76582 | ||
|
13d5cd1a58 | ||
|
08dcc7d820 | ||
|
7755563b65 | ||
|
b43cbc050d | ||
|
db7a26d47d | ||
|
92166452c1 | ||
|
96fa3ef381 | ||
|
755415c7d8 | ||
|
b168e1de55 | ||
|
38eed14b41 | ||
|
2c993f8c32 | ||
|
65858a4c74 | ||
|
263c63e009 | ||
|
813bce4697 | ||
|
a5d2237435 | ||
|
ab6d322eca | ||
|
1a8a0df107 | ||
|
d22931cb49 | ||
|
9cc2700160 | ||
|
a5a0f45dde | ||
|
3c02fa7e36 | ||
|
0d0f836f76 | ||
|
69c93cc665 | ||
|
97576e473d | ||
|
1e324e0e8d | ||
|
7ab4e1fbdb | ||
|
abecb21671 | ||
|
8be6369bdf | ||
|
0a37709b0a | ||
|
98cd1417df | ||
|
8941417a54 | ||
|
460069ceaa | ||
|
316aa44f23 | ||
|
cbcfd53dae | ||
|
df2ca51f4a | ||
|
4f4069c6b5 | ||
|
7133540c23 | ||
|
590231741d | ||
|
a9349b8f3d | ||
|
6aa1531972 | ||
|
4fcb385dc0 | ||
|
260ddedbfd | ||
|
f8917aaf88 | ||
|
c8550d5c9e | ||
|
1aeb09cb8b | ||
|
f575234fc8 | ||
|
abcd10628a | ||
|
eb2b63ce9c | ||
|
805e17988e | ||
|
80df5dcc95 | ||
|
2bc316d6f0 | ||
|
195bbc746b | ||
|
0f56b4f476 | ||
|
94719918d4 | ||
|
f2a3721d7d | ||
|
79863ae122 | ||
|
d7f898b4fb | ||
|
fe36c9e9f6 | ||
|
76ae1592d0 | ||
|
31d387749b | ||
|
9e3026bd78 | ||
|
bfe7e7777d | ||
|
81b4941fda | ||
|
0f719d41fd | ||
|
766451cbb9 | ||
|
1e4dc52197 | ||
|
6073c77921 | ||
|
b6e53205dd | ||
|
ee6dddd25a |
@@ -3,11 +3,7 @@ language: python
|
||||
sudo: false
|
||||
|
||||
python:
|
||||
- '3.5'
|
||||
- '2.7.12'
|
||||
- '2.6'
|
||||
- '3.3'
|
||||
- '3.4'
|
||||
- '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
|
||||
|
||||
install:
|
||||
- pip install -r requirements.txt
|
||||
|
@@ -30,7 +30,7 @@ another clever idea: using table-driven semantics routines, using
|
||||
format specifiers.
|
||||
|
||||
The last mention of a release of SPARK from John is around 2002. As
|
||||
released, although the Early Algorithm parser was in good shape, this
|
||||
released, although the Earley Algorithm parser was in good shape, this
|
||||
code was woefully lacking as serious Python deparser.
|
||||
|
||||
In the fall of 2000, Hartmut Goebel
|
||||
@@ -135,9 +135,9 @@ Hartmut a decade an a half ago:
|
||||
NB. This is not a masterpiece of software, but became more like a hack.
|
||||
Probably a complete rewrite would be sensefull. hG/2000-12-27
|
||||
|
||||
This project deparses using an Early-algorithm parse with lots of
|
||||
This project deparses using an Earley-algorithm parse with lots of
|
||||
massaging of tokens and the grammar in the scanner
|
||||
phase. Early-algorithm parsers are context free and tend to be linear
|
||||
phase. Earley-algorithm parsers are context free and tend to be linear
|
||||
if the grammar is LR or left recursive.
|
||||
|
||||
Another approach that doesn't use grammars is to do something like
|
||||
|
@@ -2,10 +2,16 @@ include README.rst
|
||||
include ChangeLog
|
||||
include HISTORY.md
|
||||
include LICENSE
|
||||
include Makefile
|
||||
include requirements.txt
|
||||
include requirements-dev.txt
|
||||
include DECOMPYLE-2.4-CHANGELOG.txt
|
||||
include __pkginfo__.py
|
||||
recursive-include uncompyle6 *.py
|
||||
include bin/uncompyle6
|
||||
include bin/pydisassemble
|
||||
include pytest/Makefile
|
||||
include test/Makefile
|
||||
recursive-include test *.py *.pyc
|
||||
recursive-include pytest *.py
|
||||
recursive-include pytest/testdata *
|
||||
|
4
Makefile
4
Makefile
@@ -37,7 +37,7 @@ check-3.0 check-3.1 check-3.2 check-3.5 check-3.6:
|
||||
$(MAKE) -C test $@
|
||||
|
||||
#:Tests for Python 2.6 (doesn't have pytest)
|
||||
check-2.6:
|
||||
check-2.4 check-2.5 check-2.6:
|
||||
$(MAKE) -C test $@
|
||||
|
||||
#:PyPy 2.6.1 or PyPy 5.0.1
|
||||
@@ -59,7 +59,7 @@ clean: clean_pyc
|
||||
|
||||
#: Create source (tarball) and wheel distribution
|
||||
dist:
|
||||
$(PYTHON) ./setup.py sdist bdist_wheel
|
||||
$(PYTHON) ./setup.py sdist bdist_egg
|
||||
|
||||
#: Remove .pyc files
|
||||
clean_pyc:
|
||||
|
64
NEWS
64
NEWS
@@ -1,3 +1,67 @@
|
||||
uncompyle6 2.9.10 2016-02-25
|
||||
|
||||
- Python grammar rule fixes
|
||||
- Add ability to get grammar coverage on runs
|
||||
- Handle Python 3.6 opcode BUILD_CONST_KEYMAP
|
||||
|
||||
uncompyle6 2.9.9 2016-12-16
|
||||
|
||||
- Remaining Python 3.5 ops handled
|
||||
(this also means more Python 3.6 ops are handled)
|
||||
- Python 3.5 and 3.6 async and await handled
|
||||
- Python 3.0 decompilation improved
|
||||
- Python 3 annotations fixed
|
||||
- Better control-flow detection
|
||||
- Code cleanups and misc bug fixes
|
||||
|
||||
uncompyle6 2.9.8 2016-12-16
|
||||
|
||||
- Better control-flow detection
|
||||
- pseudo instruction THEN in 2.x
|
||||
to disambiguate if from and
|
||||
- fix bug in --verify option
|
||||
- DRY (a little) control-flow detection
|
||||
- fix syntax in tuples with one element
|
||||
- if AST rule inheritence in Python 2.5
|
||||
- NAME_MODULE removal for Python <= 2.4
|
||||
- verifycall fixes for Python <= 2.4
|
||||
- more Python lint
|
||||
|
||||
uncompyle6 2.9.7 2016-12-16
|
||||
|
||||
- Start to handle 3.5/3.6 build_map_unpack_with_call
|
||||
- Some Python 3.6 bytecode to wordcode conversion fixes
|
||||
- option -g: show start-end range when possible
|
||||
- track print_docstring move to help (used in python 3.1)
|
||||
- verify: allow RETURN_VALUE to match RETURN_END_IF
|
||||
- some 3.2 compatibility
|
||||
- Better Python 3 control flow detection by adding Pseudo ELSE opcodes
|
||||
|
||||
uncompyle6 2.9.6 2016-12-04
|
||||
|
||||
- Shorten Python3 grammars with + and *
|
||||
this requires spark parser 1.5.1
|
||||
- Add some AST reduction checks to improve
|
||||
decompile accuracy. This too requires
|
||||
spark parser 1.5.1
|
||||
|
||||
uncompyle6 2.9.6 2016-11-20
|
||||
|
||||
- Correct MANIFEST.in
|
||||
- More AST grammar checking
|
||||
- --linemapping option or linenumbers.line_number_mapping()
|
||||
Shows correspondence of lines between source
|
||||
and decompiled source
|
||||
- Some control flow adjustments in code for 2.x.
|
||||
This is probably an improvement in 2.6 and before.
|
||||
For 2.7 things are just shuffled around a little. Sigh.
|
||||
Overall I think we are getting more precise in
|
||||
or analysis even if it is not always reflected
|
||||
in the results.
|
||||
- better control flow debugging output
|
||||
- Python 2 and 3 detect structure code is more similar
|
||||
- Handle Docstrings with embedded tiple quotes (""")
|
||||
|
||||
uncompyle6 2.9.5 2016-11-13
|
||||
|
||||
- Fix Python 3 bugs:
|
||||
|
54
README.rst
54
README.rst
@@ -44,8 +44,9 @@ Requirements
|
||||
------------
|
||||
|
||||
This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1.
|
||||
Python versions 2.4-2.7 are supported in the python-2.4 branch.
|
||||
The bytecode files it can read has been tested on Python bytecodes from
|
||||
versions 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
|
||||
versions 1.5, 2.1-2.7, and 3.2-3.6 and the above-mentioned PyPy versions.
|
||||
|
||||
Installation
|
||||
------------
|
||||
@@ -92,39 +93,58 @@ For usage help:
|
||||
|
||||
$ uncompyle6 -h
|
||||
|
||||
If you want strong verification of the correctness of the
|
||||
decompilation process, add the `--verify` option. But there are
|
||||
situations where this will indicate a failure, although the generated
|
||||
program is semantically equivalent. Using option `--weak-verify` will
|
||||
tell you if there is something definitely wrong. Generally, large
|
||||
swaths of code are decompiled correctly, if not the entire program.
|
||||
|
||||
You can also cross compare the results with pycdc_ . Since they work
|
||||
differently, bugs here often aren't in that, and vice versa.
|
||||
|
||||
|
||||
Known Bugs/Restrictions
|
||||
-----------------------
|
||||
|
||||
The biggest known and possibly fixable (but hard) problem has to do
|
||||
with handling control flow. In some cases we can detect an erroneous
|
||||
with handling control flow. All of the Python decompilers I have looked
|
||||
at have the same problem. In some cases we can detect an erroneous
|
||||
decompilation and report that.
|
||||
|
||||
About 90% of the decompilation verifies from Python 2.3.7 to Python
|
||||
3.4.2 on the standard library packages I have on my system.
|
||||
About 90% of the decompilation of Python standard library packages in
|
||||
Python 2.7.12 verifies correctly. Over 99% of Python 2.7 and 3.3-3.5
|
||||
"weakly" verify. Python 2.6 drops down to 96% weakly verifying.
|
||||
Other versions drop off in quality too.
|
||||
|
||||
*Verification* is the process of decompiling bytecode, compiling with
|
||||
a Python for that byecode version, and then comparing the bytecode
|
||||
a Python for that bytecode version, and then comparing the bytecode
|
||||
produced by the decompiled/compiled program. Some allowance is made
|
||||
for inessential differences. But other semantically equivalent
|
||||
differences are not caught. For example ``if x: foo()`` is
|
||||
equivalent to ``x and foo()`` and decompilation may turn one into the
|
||||
other. *Weak Verification* on the other hand doesn't check bytecode
|
||||
for equivalence but does check to see if the resulting decompiled
|
||||
source is a valid Python program by running the Python
|
||||
interpreter. Because the Python language has changed so much, for best
|
||||
results you should use the same Python Version in checking as used in
|
||||
the bytecode.
|
||||
differences are not caught. For example ``1 and 0`` is decompiled to
|
||||
the equivalent ``0``; remnants of the first true evaluation (1) is
|
||||
lost when Python compiles this. When Python next compiles ``0`` the
|
||||
resulting code is simpler.
|
||||
|
||||
*Weak Verification*
|
||||
on the other hand doesn't check bytecode for equivalence but does
|
||||
check to see if the resulting decompiled source is a valid Python
|
||||
program by running the Python interpreter. Because the Python language
|
||||
has changed so much, for best results you should use the same Python
|
||||
Version in checking as used in the bytecode.
|
||||
|
||||
Later distributions average about 200 files. There is some work to do
|
||||
on the lower end Python versions which is more difficult for us to
|
||||
handle since we don't have a Python interpreter for versions 1.5, 1.6,
|
||||
and 2.0.
|
||||
|
||||
Python 3.0 support is weak; Python 3.5 largely works, but still has
|
||||
some bugs in it. Python 3.6 changes things drastically by using word
|
||||
codes rather than byte codes. That has been addressed, but then it also
|
||||
changes function call opcodes and its semantics.
|
||||
In the Python 3 series, Python support is is strongest around 3.4 or
|
||||
3.3 and drops off as you move further away from those versions. Python
|
||||
3.5 largely works, but still has some bugs in it and is missing some
|
||||
opcodes. Python 3.6 changes things drastically by using word codes
|
||||
rather than byte codes. That has been addressed, but then it also
|
||||
changes function call opcodes and its semantics and has more problems
|
||||
with control flow than 3.5 has.
|
||||
|
||||
Currently not all Python magic numbers are supported. Specifically in
|
||||
some versions of Python, notably Python 3.6, the magic number has
|
||||
|
@@ -9,17 +9,19 @@
|
||||
|
||||
# Things that change more often go here.
|
||||
copyright = """
|
||||
Copyright (C) 2015, 2016 Rocky Bernstein <rb@dustyfeet.com>.
|
||||
Copyright (C) 2015-2017 Rocky Bernstein <rb@dustyfeet.com>.
|
||||
"""
|
||||
|
||||
classifiers = ['Development Status :: 4 - Beta',
|
||||
classifiers = ['Development Status :: 5 - Production/Stable',
|
||||
'Intended Audience :: Developers',
|
||||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 2',
|
||||
'Programming Language :: Python :: 2.4',
|
||||
'Programming Language :: Python :: 2.5',
|
||||
'Programming Language :: Python :: 2.6',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.1',
|
||||
'Programming Language :: Python :: 3.2',
|
||||
'Programming Language :: Python :: 3.3',
|
||||
'Programming Language :: Python :: 3.4',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
@@ -37,8 +39,8 @@ entry_points={
|
||||
'pydisassemble=uncompyle6.bin.pydisassemble:main',
|
||||
]}
|
||||
ftp_url = None
|
||||
install_requires = ['spark-parser >= 1.4.0, < 1.5.0',
|
||||
'xdis >= 3.2.3, < 3.3.0']
|
||||
install_requires = ['spark-parser >= 1.6.0, < 1.7.0',
|
||||
'xdis >= 3.2.4, < 3.3.0']
|
||||
license = 'MIT'
|
||||
mailing_list = 'python-debugger@googlegroups.com'
|
||||
modname = 'uncompyle6'
|
||||
|
@@ -1,6 +1,6 @@
|
||||
machine:
|
||||
python:
|
||||
version: 2.7.8
|
||||
version: 2.7.10
|
||||
environment:
|
||||
COMPILE: --compile
|
||||
|
||||
@@ -10,4 +10,4 @@ dependencies:
|
||||
- pip install -r requirements-dev.txt
|
||||
test:
|
||||
override:
|
||||
- python ./setup.py develop && make check-2.7
|
||||
- python ./setup.py develop && make check-2.6
|
||||
|
1
pytest/.gitignore
vendored
1
pytest/.gitignore
vendored
@@ -1 +1,2 @@
|
||||
/.hypothesis
|
||||
/__pycache__
|
||||
|
21
pytest/test_build_const_key_map.py
Normal file
21
pytest/test_build_const_key_map.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import pytest
|
||||
# uncompyle6
|
||||
from uncompyle6 import PYTHON_VERSION
|
||||
from validate import validate_uncompyle
|
||||
|
||||
|
||||
@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
|
||||
@pytest.mark.parametrize('text', (
|
||||
"{0.: 'a', -1: 'b'}", # BUILD_MAP
|
||||
"{'a':'b'}", # BUILD_MAP
|
||||
"{0: 1}", # BUILD_MAP
|
||||
"{b'0':1, b'2':3}", # BUILD_CONST_KEY_MAP
|
||||
"{0: 1, 2: 3}", # BUILD_CONST_KEY_MAP
|
||||
"{'a':'b','c':'d'}", # BUILD_CONST_KEY_MAP
|
||||
"{0: 1, 2: 3}", # BUILD_CONST_KEY_MAP
|
||||
"{'a': 1, 'b': 2}", # BUILD_CONST_KEY_MAP
|
||||
"{'a':'b','c':'d'}", # BUILD_CONST_KEY_MAP
|
||||
"{0.0:'b',0.1:'d'}", # BUILD_CONST_KEY_MAP
|
||||
))
|
||||
def test_build_const_key_map(text):
|
||||
validate_uncompyle(text)
|
78
pytest/test_docstring.py
Normal file
78
pytest/test_docstring.py
Normal file
@@ -0,0 +1,78 @@
|
||||
import sys
|
||||
from uncompyle6 import PYTHON3
|
||||
if PYTHON3:
|
||||
from io import StringIO
|
||||
minint = -sys.maxsize-1
|
||||
maxint = sys.maxsize
|
||||
else:
|
||||
from StringIO import StringIO
|
||||
minint = -sys.maxint-1
|
||||
maxint = sys.maxint
|
||||
from uncompyle6.semantics.helper import print_docstring
|
||||
|
||||
class PrintFake():
|
||||
def __init__(self):
|
||||
self.pending_newlines = 0
|
||||
self.f = StringIO()
|
||||
|
||||
def write(self, *data):
|
||||
if (len(data) == 0) or (len(data) == 1 and data[0] == ''):
|
||||
return
|
||||
out = ''.join((str(j) for j in data))
|
||||
n = 0
|
||||
for i in out:
|
||||
if i == '\n':
|
||||
n += 1
|
||||
if n == len(out):
|
||||
self.pending_newlines = max(self.pending_newlines, n)
|
||||
return
|
||||
elif n:
|
||||
self.pending_newlines = max(self.pending_newlines, n)
|
||||
out = out[n:]
|
||||
break
|
||||
else:
|
||||
break
|
||||
|
||||
if self.pending_newlines > 0:
|
||||
self.f.write('\n'*self.pending_newlines)
|
||||
self.pending_newlines = 0
|
||||
|
||||
for i in out[::-1]:
|
||||
if i == '\n':
|
||||
self.pending_newlines += 1
|
||||
else:
|
||||
break
|
||||
|
||||
if self.pending_newlines:
|
||||
out = out[:-self.pending_newlines]
|
||||
self.f.write(out)
|
||||
def println(self, *data):
|
||||
if data and not(len(data) == 1 and data[0] ==''):
|
||||
self.write(*data)
|
||||
self.pending_newlines = max(self.pending_newlines, 1)
|
||||
return
|
||||
pass
|
||||
|
||||
def test_docstring():
|
||||
|
||||
for doc, expect in (
|
||||
("Now is the time",
|
||||
' """Now is the time"""'),
|
||||
("""
|
||||
Now is the time
|
||||
""",
|
||||
''' """
|
||||
Now is the time
|
||||
"""''')
|
||||
|
||||
# (r'''func placeholder - ' and with ("""\nstring\n """)''',
|
||||
# """ r'''func placeholder - ' and with (\"\"\"\nstring\n\"\"\")'''"""),
|
||||
# (r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """,
|
||||
# """ r\"\"\"func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" \"\"\"""")
|
||||
):
|
||||
|
||||
o = PrintFake()
|
||||
# print(doc)
|
||||
# print(expect)
|
||||
print_docstring(o, ' ', doc)
|
||||
assert expect == o.f.getvalue()
|
@@ -8,6 +8,18 @@ def bug(state, slotstate):
|
||||
for key, value in slotstate.items():
|
||||
setattr(state, key, 2)
|
||||
|
||||
# From 2.7 disassemble
|
||||
# Problem is not getting while, because
|
||||
# COME_FROM not added
|
||||
def bug_loop(disassemble, tb=None):
|
||||
if tb:
|
||||
try:
|
||||
tb = 5
|
||||
except AttributeError:
|
||||
raise RuntimeError
|
||||
while tb: tb = tb.tb_next
|
||||
disassemble(tb)
|
||||
|
||||
def test_if_in_for():
|
||||
code = bug.__code__
|
||||
scan = get_scanner(PYTHON_VERSION)
|
||||
@@ -16,18 +28,35 @@ def test_if_in_for():
|
||||
n = scan.setup_code(code)
|
||||
scan.build_lines_data(code, n)
|
||||
scan.build_prev_op(n)
|
||||
fjt = scan.find_jump_targets()
|
||||
fjt = scan.find_jump_targets(False)
|
||||
assert {15: [3], 69: [66], 63: [18]} == fjt
|
||||
assert scan.structs == \
|
||||
[{'start': 0, 'end': 72, 'type': 'root'},
|
||||
{'start': 18, 'end': 66, 'type': 'if-then'},
|
||||
{'start': 15, 'end': 66, 'type': 'if-then'},
|
||||
{'start': 31, 'end': 59, 'type': 'for-loop'},
|
||||
{'start': 62, 'end': 63, 'type': 'for-else'}]
|
||||
|
||||
code = bug_loop.__code__
|
||||
n = scan.setup_code(code)
|
||||
scan.build_lines_data(code, n)
|
||||
scan.build_prev_op(n)
|
||||
fjt = scan.find_jump_targets(False)
|
||||
assert{64: [42], 67: [42, 42], 42: [16, 41], 19: [6]} == fjt
|
||||
assert scan.structs == [
|
||||
{'start': 0, 'end': 80, 'type': 'root'},
|
||||
{'start': 3, 'end': 64, 'type': 'if-then'},
|
||||
{'start': 6, 'end': 15, 'type': 'try'},
|
||||
{'start': 19, 'end': 38, 'type': 'except'},
|
||||
{'start': 45, 'end': 67, 'type': 'while-loop'},
|
||||
{'start': 70, 'end': 64, 'type': 'while-else'},
|
||||
# previous bug was not mistaking while-loop for if-then
|
||||
{'start': 48, 'end': 67, 'type': 'while-loop'}]
|
||||
|
||||
elif 3.2 < PYTHON_VERSION <= 3.4:
|
||||
scan.code = array('B', code.co_code)
|
||||
scan.build_lines_data(code)
|
||||
scan.build_prev_op()
|
||||
fjt = scan.find_jump_targets()
|
||||
fjt = scan.find_jump_targets(False)
|
||||
assert {69: [66], 63: [18]} == fjt
|
||||
assert scan.structs == \
|
||||
[{'end': 72, 'type': 'root', 'start': 0},
|
||||
|
@@ -1,150 +0,0 @@
|
||||
# std
|
||||
import os
|
||||
# test
|
||||
import pytest
|
||||
import hypothesis
|
||||
from hypothesis import strategies as st
|
||||
# uncompyle6
|
||||
from uncompyle6 import PYTHON_VERSION, deparse_code
|
||||
|
||||
|
||||
@st.composite
|
||||
def expressions(draw):
|
||||
# todo : would be nice to generate expressions using hypothesis however
|
||||
# this is pretty involved so for now just use a corpus of expressions
|
||||
# from which to select.
|
||||
return draw(st.sampled_from((
|
||||
'abc',
|
||||
'len(items)',
|
||||
'x + 1',
|
||||
'lineno',
|
||||
'container',
|
||||
'self.attribute',
|
||||
'self.method()',
|
||||
# These expressions are failing, I think these are control
|
||||
# flow problems rather than problems with FORMAT_VALUE,
|
||||
# however I need to confirm this...
|
||||
#'sorted(items, key=lambda x: x.name)',
|
||||
#'func(*args, **kwargs)',
|
||||
#'text or default',
|
||||
#'43 if life_the_universe and everything else None'
|
||||
)))
|
||||
|
||||
|
||||
@st.composite
|
||||
def format_specifiers(draw):
|
||||
"""
|
||||
Generate a valid format specifier using the rules:
|
||||
|
||||
format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]
|
||||
fill ::= <any character>
|
||||
align ::= "<" | ">" | "=" | "^"
|
||||
sign ::= "+" | "-" | " "
|
||||
width ::= integer
|
||||
precision ::= integer
|
||||
type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"
|
||||
|
||||
See https://docs.python.org/2/library/string.html
|
||||
|
||||
:param draw: Let hypothesis draw from other strategies.
|
||||
|
||||
:return: An example format_specifier.
|
||||
"""
|
||||
alphabet_strategy = st.characters(min_codepoint=ord('a'), max_codepoint=ord('z'))
|
||||
fill = draw(st.one_of(alphabet_strategy, st.none()))
|
||||
align = draw(st.sampled_from(list('<>=^')))
|
||||
fill_align = (fill + align or '') if fill else ''
|
||||
|
||||
type_ = draw(st.sampled_from('bcdeEfFgGnosxX%'))
|
||||
can_have_sign = type_ in 'deEfFgGnoxX%'
|
||||
can_have_comma = type_ in 'deEfFgG%'
|
||||
can_have_precision = type_ in 'fFgG'
|
||||
can_have_pound = type_ in 'boxX%'
|
||||
can_have_zero = type_ in 'oxX'
|
||||
|
||||
sign = draw(st.sampled_from(list('+- ') + [''])) if can_have_sign else ''
|
||||
pound = draw(st.sampled_from(('#', '',))) if can_have_pound else ''
|
||||
zero = draw(st.sampled_from(('0', '',))) if can_have_zero else ''
|
||||
|
||||
int_strategy = st.integers(min_value=1, max_value=1000)
|
||||
|
||||
width = draw(st.one_of(int_strategy, st.none()))
|
||||
width = str(width) if width is not None else ''
|
||||
|
||||
comma = draw(st.sampled_from((',', '',))) if can_have_comma else ''
|
||||
if can_have_precision:
|
||||
precision = draw(st.one_of(int_strategy, st.none()))
|
||||
precision = '.' + str(precision) if precision else ''
|
||||
else:
|
||||
precision = ''
|
||||
|
||||
return ''.join((fill_align, sign, pound, zero, width, comma, precision, type_,))
|
||||
|
||||
|
||||
@st.composite
|
||||
def fstrings(draw):
|
||||
"""
|
||||
Generate a valid f-string.
|
||||
See https://www.python.org/dev/peps/pep-0498/#specification
|
||||
|
||||
:param draw: Let hypothsis draw from other strategies.
|
||||
|
||||
:return: A valid f-string.
|
||||
"""
|
||||
character_strategy = st.characters(
|
||||
blacklist_characters='\r\n\'\\s{}',
|
||||
min_codepoint=1,
|
||||
max_codepoint=1000,
|
||||
)
|
||||
is_raw = draw(st.booleans())
|
||||
integer_strategy = st.integers(min_value=0, max_value=3)
|
||||
expression_count = draw(integer_strategy)
|
||||
content = []
|
||||
for _ in range(expression_count):
|
||||
expression = draw(expressions())
|
||||
conversion = draw(st.sampled_from(('', '!s', '!r', '!a',)))
|
||||
has_specifier = draw(st.booleans())
|
||||
specifier = ':' + draw(format_specifiers()) if has_specifier else ''
|
||||
content.append('{{{}{}}}'.format(expression, conversion, specifier))
|
||||
content.append(draw(st.text(character_strategy)))
|
||||
content = ''.join(content)
|
||||
return "f{}'{}'".format('r' if is_raw else '', content)
|
||||
|
||||
|
||||
@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
|
||||
@hypothesis.given(format_specifiers())
|
||||
def test_format_specifiers(format_specifier):
|
||||
"""Verify that format_specifiers generates valid specifiers"""
|
||||
try:
|
||||
exec('"{:' + format_specifier + '}".format(0)')
|
||||
except ValueError as e:
|
||||
if 'Unknown format code' not in str(e):
|
||||
raise
|
||||
|
||||
|
||||
def run_test(text):
|
||||
hypothesis.assume(len(text))
|
||||
hypothesis.assume("f'{" in text)
|
||||
expr = text + '\n'
|
||||
code = compile(expr, '<string>', 'single')
|
||||
deparsed = deparse_code(PYTHON_VERSION, code, compile_mode='single')
|
||||
recompiled = compile(deparsed.text, '<string>', 'single')
|
||||
if recompiled != code:
|
||||
assert 'dis(' + deparsed.text.strip('\n') + ')' == 'dis(' + expr.strip('\n') + ')'
|
||||
|
||||
|
||||
@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
|
||||
@hypothesis.given(fstrings())
|
||||
def test_uncompyle_fstring(fstring):
|
||||
"""Verify uncompyling fstring bytecode"""
|
||||
run_test(fstring)
|
||||
|
||||
|
||||
@pytest.mark.skipif(PYTHON_VERSION < 3.6, reason='need at least python 3.6')
|
||||
@pytest.mark.parametrize('fstring', [
|
||||
"f'{abc}{abc!s}'",
|
||||
"f'{abc}0'",
|
||||
])
|
||||
def test_uncompyle_direct(fstring):
|
||||
"""useful for debugging"""
|
||||
run_test(fstring)
|
@@ -1,6 +1,6 @@
|
||||
import re
|
||||
from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
|
||||
from uncompyle6.parser import get_python_parser
|
||||
from uncompyle6.parser import get_python_parser, python_parser
|
||||
from uncompyle6.scanner import get_scanner
|
||||
|
||||
def test_grammar():
|
||||
@@ -41,7 +41,7 @@ def test_grammar():
|
||||
"""
|
||||
JUMP_BACK CONTINUE RETURN_END_IF
|
||||
COME_FROM COME_FROM_EXCEPT COME_FROM_LOOP COME_FROM_WITH
|
||||
COME_FROM_FINALLY
|
||||
COME_FROM_FINALLY ELSE
|
||||
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
|
||||
LAMBDA_MARKER RETURN_LAST
|
||||
""".split())
|
||||
@@ -53,3 +53,11 @@ def test_grammar():
|
||||
ignore_set.add('STORE_LOCALS')
|
||||
opcode_set = set(s.opc.opname).union(ignore_set)
|
||||
check_tokens(tokens, opcode_set)
|
||||
|
||||
def test_dup_rule():
|
||||
import inspect
|
||||
python_parser(PYTHON_VERSION, inspect.currentframe().f_code,
|
||||
is_pypy=IS_PYPY,
|
||||
parser_debug={
|
||||
'dups': True, 'transition': False, 'reduce': False,
|
||||
'rules': False, 'errorstack': None, 'context': True})
|
||||
|
143
pytest/validate.py
Normal file
143
pytest/validate.py
Normal file
@@ -0,0 +1,143 @@
|
||||
# future
|
||||
from __future__ import print_function
|
||||
# std
|
||||
import os
|
||||
import dis
|
||||
import difflib
|
||||
import subprocess
|
||||
import tempfile
|
||||
# compatability
|
||||
import six
|
||||
# uncompyle6 / xdis
|
||||
from uncompyle6 import PYTHON_VERSION, deparse_code
|
||||
|
||||
|
||||
def _dis_to_text(co):
|
||||
return dis.Bytecode(co).dis()
|
||||
|
||||
|
||||
def print_diff(original, uncompyled):
|
||||
"""
|
||||
Try and display a pretty html line difference between the original and
|
||||
uncompyled code and bytecode if elinks and BeautifulSoup are installed
|
||||
otherwise just show the diff.
|
||||
|
||||
:param original: Text describing the original code object.
|
||||
:param uncompyled: Text describing the uncompyled code object.
|
||||
"""
|
||||
original_lines = original.split('\n')
|
||||
uncompyled_lines = uncompyled.split('\n')
|
||||
args = original_lines, uncompyled_lines, 'original', 'uncompyled'
|
||||
try:
|
||||
from bs4 import BeautifulSoup
|
||||
diff = difflib.HtmlDiff().make_file(*args)
|
||||
diff = BeautifulSoup(diff, "html.parser")
|
||||
diff.select_one('table[summary="Legends"]').extract()
|
||||
except ImportError:
|
||||
print('\nTo display diff highlighting run:\n pip install BeautifulSoup4')
|
||||
diff = difflib.HtmlDiff().make_table(*args)
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False) as f:
|
||||
f.write(str(diff).encode('utf-8'))
|
||||
|
||||
try:
|
||||
print()
|
||||
html = subprocess.check_output([
|
||||
'elinks',
|
||||
'-dump',
|
||||
'-no-references',
|
||||
'-dump-color-mode',
|
||||
'1',
|
||||
f.name,
|
||||
]).decode('utf-8')
|
||||
print(html)
|
||||
except:
|
||||
print('\nFor side by side diff install elinks')
|
||||
diff = difflib.Differ().compare(original_lines, uncompyled_lines)
|
||||
print('\n'.join(diff))
|
||||
finally:
|
||||
os.unlink(f.name)
|
||||
|
||||
|
||||
def are_instructions_equal(i1, i2):
|
||||
"""
|
||||
Determine if two instructions are approximately equal,
|
||||
ignoring certain fields which we allow to differ, namely:
|
||||
|
||||
* code objects are ignore (should probaby be checked) due to address
|
||||
* line numbers
|
||||
|
||||
:param i1: left instruction to compare
|
||||
:param i2: right instruction to compare
|
||||
|
||||
:return: True if the two instructions are approximately equal, otherwise False.
|
||||
"""
|
||||
result = (1==1
|
||||
and i1.opname == i2.opname
|
||||
and i1.opcode == i2.opcode
|
||||
and i1.arg == i2.arg
|
||||
# ignore differences due to code objects
|
||||
# TODO : Better way of ignoring address
|
||||
and (i1.argval == i2.argval or '<code object' in str(i1.argval))
|
||||
# TODO : Should probably recurse to check code objects
|
||||
and (i1.argrepr == i2.argrepr or '<code object' in i1.argrepr)
|
||||
and i1.offset == i2.offset
|
||||
# ignore differences in line numbers
|
||||
#and i1.starts_line
|
||||
and i1.is_jump_target == i2.is_jump_target
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def are_code_objects_equal(co1, co2):
|
||||
"""
|
||||
Determine if two code objects are approximately equal,
|
||||
see are_instructions_equal for more information.
|
||||
|
||||
:param i1: left code object to compare
|
||||
:param i2: right code object to compare
|
||||
|
||||
:return: True if the two code objects are approximately equal, otherwise False.
|
||||
"""
|
||||
# TODO : Use xdis for python2 compatability
|
||||
instructions1 = dis.Bytecode(co1)
|
||||
instructions2 = dis.Bytecode(co2)
|
||||
for opcode1, opcode2 in zip(instructions1, instructions2):
|
||||
if not are_instructions_equal(opcode1, opcode2):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def validate_uncompyle(text, mode='exec'):
|
||||
"""
|
||||
Validate decompilation of the given source code.
|
||||
|
||||
:param text: Source to validate decompilation of.
|
||||
"""
|
||||
original_code = compile(text, '<string>', mode)
|
||||
original_dis = _dis_to_text(original_code)
|
||||
original_text = text
|
||||
|
||||
deparsed = deparse_code(PYTHON_VERSION, original_code,
|
||||
compile_mode=mode, out=six.StringIO())
|
||||
uncompyled_text = deparsed.text
|
||||
uncompyled_code = compile(uncompyled_text, '<string>', 'exec')
|
||||
|
||||
if not are_code_objects_equal(uncompyled_code, original_code):
|
||||
|
||||
uncompyled_dis = _dis_to_text(uncompyled_text)
|
||||
|
||||
def output(text, dis):
|
||||
width = 60
|
||||
return '\n\n'.join([
|
||||
' SOURCE CODE '.center(width, '#'),
|
||||
text.strip(),
|
||||
' BYTECODE '.center(width, '#'),
|
||||
dis
|
||||
])
|
||||
|
||||
original = output(original_text, original_dis)
|
||||
uncompyled = output(uncompyled_text, uncompyled_dis)
|
||||
print_diff(original, uncompyled)
|
||||
|
||||
assert 'original' == 'uncompyled'
|
@@ -1,3 +1,4 @@
|
||||
pytest
|
||||
flake8
|
||||
hypothesis
|
||||
hypothesis
|
||||
six
|
2
setup.py
2
setup.py
@@ -24,6 +24,6 @@ setup(
|
||||
py_modules = py_modules,
|
||||
test_suite = 'nose.collector',
|
||||
url = web,
|
||||
setup_requires = ['nose>=1.0'],
|
||||
tests_require = ['nose>=1.0'],
|
||||
version = VERSION,
|
||||
zip_safe = zip_safe)
|
||||
|
@@ -20,11 +20,11 @@ check:
|
||||
$(MAKE) check-$$PYTHON_VERSION
|
||||
|
||||
#: Run working tests from Python 2.6 or 2.7
|
||||
check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-2.7-ok
|
||||
check-2.4 check-2.5 check-2.6 check-2.7: check-bytecode-2 check-bytecode-3 check-bytecode-1 check-2.7-ok
|
||||
|
||||
#: Run working tests from Python 3.1
|
||||
#: Run working tests from Python 3.0
|
||||
check-3.0: check-bytecode
|
||||
@echo Python 3.0 testing not done yet
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.0 --weak-verify $(COMPILE)
|
||||
|
||||
#: Run working tests from Python 3.1
|
||||
check-3.1: check-bytecode
|
||||
@@ -36,11 +36,11 @@ check-3.2: check-bytecode
|
||||
|
||||
#: Run working tests from Python 3.3
|
||||
check-3.3: check-bytecode
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.3 --weak-verify $(COMPILE)
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.3 --verify $(COMPILE)
|
||||
|
||||
#: Run working tests from Python 3.4
|
||||
check-3.4: check-bytecode check-3.4-ok check-2.7-ok
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.4 --weak-verify $(COMPILE)
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.4 --verify $(COMPILE)
|
||||
|
||||
#: Run working tests from Python 3.5
|
||||
check-3.5: check-bytecode
|
||||
@@ -98,14 +98,6 @@ check-bytecode-2.4:
|
||||
check-bytecode-2.5:
|
||||
$(PYTHON) test_pythonlib.py --bytecode-2.5
|
||||
|
||||
#: Check deparsing Python 2.6
|
||||
check-bytecode-2.6:
|
||||
$(PYTHON) test_pythonlib.py --bytecode-2.6
|
||||
|
||||
#: Check deparsing Python 2.7
|
||||
check-bytecode-2.7:
|
||||
$(PYTHON) test_pythonlib.py --bytecode-2.7
|
||||
|
||||
#: Check deparsing Python 3.0
|
||||
check-bytecode-3.0:
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.0
|
||||
@@ -134,6 +126,26 @@ check-bytecode-3.5:
|
||||
check-bytecode-3.6:
|
||||
$(PYTHON) test_pythonlib.py --bytecode-3.6
|
||||
|
||||
#: Get grammar coverage for Python 2.4
|
||||
grammar-coverage-2.4:
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pythonlib.py --bytecode-2.4
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-24.cover $(PYTHON) test_pyenvlib.py --2.4.6
|
||||
|
||||
#: Get grammar coverage for Python 2.5
|
||||
grammar-coverage-2.5:
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pythonlib.py --bytecode-2.5
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-25.cover $(PYTHON) test_pyenvlib.py --2.5.6
|
||||
|
||||
#: Get grammar coverage for Python 2.6
|
||||
grammar-coverage-2.6:
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pythonlib.py --bytecode-2.6
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-26.cover $(PYTHON) test_pyenvlib.py --2.6.9
|
||||
|
||||
#: Get grammar coverage for Python 2.7
|
||||
grammar-coverage-2.7:
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pythonlib.py --bytecode-2.7
|
||||
SPARK_PARSER_COVERAGE=/tmp/spark-grammar-27.cover $(PYTHON) test_pyenvlib.py --2.7.13
|
||||
|
||||
#: short tests for bytecodes only for this version of Python
|
||||
check-native-short:
|
||||
$(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE)
|
||||
|
BIN
test/bytecode_2.4/01_ops.pyc
Normal file
BIN
test/bytecode_2.4/01_ops.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.5/01_ops.pyc
Normal file
BIN
test/bytecode_2.5/01_ops.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.5/02_try_else.pyc
Normal file
BIN
test/bytecode_2.5/02_try_else.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
test/bytecode_2.6/01_ops.pyc
Normal file
BIN
test/bytecode_2.6/01_ops.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.6/03_elif_vs_continue.pyc
Normal file
BIN
test/bytecode_2.6/03_elif_vs_continue.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.6/03_if_for.pyc
Normal file
BIN
test/bytecode_2.6/03_if_for.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.6/03_if_vs_and.pyc
Normal file
BIN
test/bytecode_2.6/03_if_vs_and.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.6/04_if_and_bug.pyc
Normal file
BIN
test/bytecode_2.6/04_if_and_bug.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
test/bytecode_2.7/00_docstring.pyc
Normal file
BIN
test/bytecode_2.7/00_docstring.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_2.7/01_ops.pyc
Normal file
BIN
test/bytecode_2.7/01_ops.pyc
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
test/bytecode_3.3/03_while_else.pyc
Normal file
BIN
test/bytecode_3.3/03_while_else.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
test/bytecode_3.4/05_while_true_break.pyc
Normal file
BIN
test/bytecode_3.4/05_while_true_break.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.5/02_build_map_unpack_with_call.pyc
Normal file
BIN
test/bytecode_3.5/02_build_map_unpack_with_call.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.5/02_try_finally.pyc
Normal file
BIN
test/bytecode_3.5/02_try_finally.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.5/03_async_await.pyc
Normal file
BIN
test/bytecode_3.5/03_async_await.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.5/03_double_star_unpack.pyc
Normal file
BIN
test/bytecode_3.5/03_double_star_unpack.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.5/03_while-if-break.pyc
Normal file
BIN
test/bytecode_3.5/03_while-if-break.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.5/04_def_annotate.pyc
Normal file
BIN
test/bytecode_3.5/04_def_annotate.pyc
Normal file
Binary file not shown.
Binary file not shown.
BIN
test/bytecode_3.6/00_assign.pyc
Normal file
BIN
test/bytecode_3.6/00_assign.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/00_docstring.pyc
Normal file
BIN
test/bytecode_3.6/00_docstring.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/00_import.pyc
Normal file
BIN
test/bytecode_3.6/00_import.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/01_matrix_multiply.pyc
Normal file
BIN
test/bytecode_3.6/01_matrix_multiply.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/02_build_map_unpack_with_call.pyc
Normal file
BIN
test/bytecode_3.6/02_build_map_unpack_with_call.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/03_async_await.pyc
Normal file
BIN
test/bytecode_3.6/03_async_await.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/03_while-if-break.pyc
Normal file
BIN
test/bytecode_3.6/03_while-if-break.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/03_while_else.pyc
Normal file
BIN
test/bytecode_3.6/03_while_else.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/04_raise.pyc
Normal file
BIN
test/bytecode_3.6/04_raise.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/05_try_finally_pass.pyc
Normal file
BIN
test/bytecode_3.6/05_try_finally_pass.pyc
Normal file
Binary file not shown.
BIN
test/bytecode_3.6/10_if_break_finally.pyc
Normal file
BIN
test/bytecode_3.6/10_if_break_finally.pyc
Normal file
Binary file not shown.
Binary file not shown.
18
test/simple_source/bug22/01_ops.py
Normal file
18
test/simple_source/bug22/01_ops.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# Statements to beef up grammar coverage rules
|
||||
# Force "inplace" ops
|
||||
y = +10 # UNARY_POSITIVE
|
||||
y /= 1 # INPLACE_DIVIDE
|
||||
y %= 4 # INPLACE_MODULO
|
||||
y **= 1 # INPLACE POWER
|
||||
y >>= 2 # INPLACE_RSHIFT
|
||||
y <<= 2 # INPLACE_LSHIFT
|
||||
y //= 1 # INPLACE_TRUE_DIVIDE
|
||||
y &= 1 # INPLACE_AND
|
||||
y ^= 1 # INPLACE_XOR
|
||||
|
||||
`y` # UNARY_CONVERT - No in Python 3.x
|
||||
|
||||
# Beef up augassign and STORE_SLICE+3
|
||||
x = [1,2,3,4,5]
|
||||
x[0:1] = 1
|
||||
x[0:3] += 1, 2, 3
|
27
test/simple_source/bug25/02_try_else.py
Normal file
27
test/simple_source/bug25/02_try_else.py
Normal file
@@ -0,0 +1,27 @@
|
||||
# Python 2.5 bug
|
||||
# Was turning into tryelse when there in fact is no else.
|
||||
def options(self, section):
|
||||
try:
|
||||
opts = self._sections[section].copy()
|
||||
except KeyError:
|
||||
raise NoSectionError(section)
|
||||
opts.update(self._defaults)
|
||||
if '__name__' in opts:
|
||||
del opts['__name__']
|
||||
return opts.keys()
|
||||
|
||||
# From python2.5/distutils/command/register.py
|
||||
def post_to_server(self, urllib2):
|
||||
try:
|
||||
result = 5
|
||||
except urllib2.HTTPError, e:
|
||||
result = e.code, e.msg
|
||||
except urllib2.URLError, e:
|
||||
result = 500
|
||||
else:
|
||||
if self.show_response:
|
||||
result = 10
|
||||
result = 200
|
||||
if self.show_response:
|
||||
result = 8
|
||||
return result
|
7
test/simple_source/bug25/03_if_for.py
Normal file
7
test/simple_source/bug25/03_if_for.py
Normal file
@@ -0,0 +1,7 @@
|
||||
# From Python 2.6. distutils/sysconfig.py
|
||||
def get_config_vars(_config_vars, args):
|
||||
if _config_vars:
|
||||
if args == 1:
|
||||
if args < 8:
|
||||
for key in ('LDFLAGS', 'BASECFLAGS'):
|
||||
_config_vars[key] = 4
|
18
test/simple_source/bug26/03_elif_vs_continue.py
Normal file
18
test/simple_source/bug26/03_elif_vs_continue.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# Bug was using continue fouling up 1st elif, by confusing
|
||||
# the "pass" for "continue" by not recognizing the if jump
|
||||
# around it. We fixed by ignoring what's done in Python 2.7
|
||||
# Better is better detection of control structures
|
||||
|
||||
def _compile_charset(charset, flags, code, fixup=None):
|
||||
# compile charset subprogram
|
||||
emit = code.append
|
||||
if fixup is None:
|
||||
fixup = 1
|
||||
for op, av in charset:
|
||||
if op is flags:
|
||||
pass
|
||||
elif op is code:
|
||||
emit(fixup(av))
|
||||
else:
|
||||
raise RuntimeError
|
||||
emit(5)
|
22
test/simple_source/bug26/03_if_vs_and.py
Normal file
22
test/simple_source/bug26/03_if_vs_and.py
Normal file
@@ -0,0 +1,22 @@
|
||||
# From 2.6 decimal
|
||||
# Bug was not recognizing scope of if and
|
||||
# turning it into xc == 1 and xe *= yc
|
||||
def _power_exact(y, xc, yc, xe):
|
||||
yc, ye = y.int, y.exp
|
||||
while yc % 10 == 0:
|
||||
yc //= 10
|
||||
ye += 1
|
||||
|
||||
if xc == 1:
|
||||
xe *= yc
|
||||
while xe % 10 == 0:
|
||||
xe //= 10
|
||||
ye += 1
|
||||
if ye < 0:
|
||||
return None
|
||||
exponent = xe * 10**ye
|
||||
if y and xe:
|
||||
xc = exponent
|
||||
else:
|
||||
xc = 0
|
||||
return 5
|
@@ -8,3 +8,12 @@ def open(file, mode = "r", buffering = None,
|
||||
encoding = None, errors = None,
|
||||
newline = None, closefd = True) -> "IOBase":
|
||||
return text
|
||||
|
||||
def foo(x: 'an argument that defaults to 5' = 5):
|
||||
print(x)
|
||||
|
||||
def div(a: dict(type=float, help='the dividend'),
|
||||
b: dict(type=float, help='the divisor (must be different than 0)')
|
||||
) -> dict(type=float, help='the result of dividing a by b'):
|
||||
"""Divide a by b"""
|
||||
return a / b
|
||||
|
14
test/simple_source/bug33/02_while1.py
Normal file
14
test/simple_source/bug33/02_while1.py
Normal file
@@ -0,0 +1,14 @@
|
||||
# From Python 3.4 mailcap
|
||||
def readmailcapfile(caps):
|
||||
while 1:
|
||||
line = 'abc'
|
||||
if line[0] == '#' or line == '':
|
||||
continue
|
||||
key, fields = (1,2)
|
||||
if not (key and fields):
|
||||
continue
|
||||
if key in caps:
|
||||
caps[key].append(fields)
|
||||
else:
|
||||
caps[key] = [fields]
|
||||
return caps
|
8
test/simple_source/bug33/03_while_else.py
Normal file
8
test/simple_source/bug33/03_while_else.py
Normal file
@@ -0,0 +1,8 @@
|
||||
# Bug from 3.4 threading. Bug is handling while/else
|
||||
def acquire(self):
|
||||
with self._cond:
|
||||
while self:
|
||||
rc = False
|
||||
else:
|
||||
rc = True
|
||||
return rc
|
@@ -0,0 +1 @@
|
||||
f(**a, **b)
|
26
test/simple_source/bug35/03_async_await.py
Normal file
26
test/simple_source/bug35/03_async_await.py
Normal file
@@ -0,0 +1,26 @@
|
||||
# Python 3.5+ async and await
|
||||
async def await_test(asyncio):
|
||||
reader, writer = await asyncio.open_connection(80)
|
||||
await bar()
|
||||
|
||||
async def afor_test():
|
||||
|
||||
async for i in [1,2,3]:
|
||||
x = i
|
||||
|
||||
|
||||
async def afor_else_test():
|
||||
|
||||
async for i in [1,2,3]:
|
||||
x = i
|
||||
else:
|
||||
z = 4
|
||||
|
||||
|
||||
async def awith_test():
|
||||
async with i:
|
||||
print(i)
|
||||
|
||||
async def awith_as_test():
|
||||
async with 1 as i:
|
||||
print(i)
|
9
test/simple_source/bug35/03_double_star_unpack.py
Normal file
9
test/simple_source/bug35/03_double_star_unpack.py
Normal file
@@ -0,0 +1,9 @@
|
||||
# Bug in Python 3.5 is getting the two star'd arguments right.
|
||||
def sum(a,b,c,d):
|
||||
return a + b + c + d
|
||||
|
||||
args=(1,2)
|
||||
sum(*args, *args)
|
||||
|
||||
# FIXME: this is handled incorrectly
|
||||
# (*c,) = (3,4)
|
7
test/simple_source/bug35/03_while-if-break.py
Normal file
7
test/simple_source/bug35/03_while-if-break.py
Normal file
@@ -0,0 +1,7 @@
|
||||
# Python 3.5 and 3.6 break inside a
|
||||
# while True and if / break
|
||||
def display_date(loop):
|
||||
while True:
|
||||
if loop.time():
|
||||
break
|
||||
x = 5
|
11
test/simple_source/exception/02_try_finally.py
Normal file
11
test/simple_source/exception/02_try_finally.py
Normal file
@@ -0,0 +1,11 @@
|
||||
# From 2.6.9 cmd.py
|
||||
try:
|
||||
if __file__:
|
||||
x = 2
|
||||
x = 3
|
||||
finally:
|
||||
if x and __file__:
|
||||
try:
|
||||
x = 1
|
||||
except:
|
||||
pass
|
@@ -5,3 +5,8 @@ def some_function():
|
||||
def some_other_function():
|
||||
some_variable, = some_function()
|
||||
print(some_variable)
|
||||
|
||||
empty_tup = ()
|
||||
one_item_tup = ("item1", )
|
||||
one_item_tup_without_parentheses = "item",
|
||||
many_items_tup = ("item1", "item2", "item3")
|
||||
|
7
test/simple_source/stmts/00_docstring.py
Normal file
7
test/simple_source/stmts/00_docstring.py
Normal file
@@ -0,0 +1,7 @@
|
||||
# uncompyle2 bug was not escaping """ properly
|
||||
r'''func placeholder - with ("""\nstring\n""")'''
|
||||
def foo():
|
||||
r'''func placeholder - ' and with ("""\nstring\n""")'''
|
||||
|
||||
def bar():
|
||||
r"""func placeholder - ' and with ('''\nstring\n''') and \"\"\"\nstring\n\"\"\" """
|
@@ -19,8 +19,6 @@ Step 2: Run the test:
|
||||
test_pyenvlib --mylib --verify # decompile verify 'mylib'
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6 import main, PYTHON3
|
||||
import os, time, shutil
|
||||
from fnmatch import fnmatch
|
||||
@@ -30,10 +28,10 @@ from fnmatch import fnmatch
|
||||
TEST_VERSIONS=('2.3.7', '2.4.6', '2.5.6', '2.6.9',
|
||||
'pypy-2.4.0', 'pypy-2.6.1',
|
||||
'pypy-5.0.1', 'pypy-5.3.1',
|
||||
'2.7.10', '2.7.11', '2.7.12',
|
||||
'2.7.10', '2.7.11', '2.7.12', '2.7.13',
|
||||
'3.0.1', '3.1.5', '3.2.6',
|
||||
'3.3.5', '3.3.6',
|
||||
'3.4.2', '3.5.1')
|
||||
'3.4.2', '3.5.1', '3.6.0')
|
||||
|
||||
target_base = '/tmp/py-dis/'
|
||||
lib_prefix = os.path.join(os.environ['HOME'], '.pyenv/versions')
|
||||
@@ -106,28 +104,40 @@ def do_tests(src_dir, patterns, target_dir, start_with=None, do_verify=False):
|
||||
if __name__ == '__main__':
|
||||
import getopt, sys
|
||||
|
||||
do_verify = False
|
||||
do_coverage = do_verify = False
|
||||
test_dirs = []
|
||||
start_with = None
|
||||
|
||||
test_options_keys = list(test_options.keys())
|
||||
test_options_keys.sort()
|
||||
opts, args = getopt.getopt(sys.argv[1:], '',
|
||||
['start-with=', 'verify', 'weak-verify', 'all', ] \
|
||||
['start-with=', 'verify', 'weak-verify',
|
||||
'coverage', 'all', ] \
|
||||
+ test_options_keys )
|
||||
vers = ''
|
||||
for opt, val in opts:
|
||||
if opt == '--verify':
|
||||
do_verify = True
|
||||
if opt == '--weak-verify':
|
||||
do_verify = 'weak'
|
||||
if opt == '--coverage':
|
||||
do_coverage = True
|
||||
elif opt == '--start-with':
|
||||
start_with = val
|
||||
elif opt[2:] in test_options_keys:
|
||||
test_dirs.append(test_options[opt[2:]])
|
||||
triple = test_options[opt[2:]]
|
||||
vers = triple[-1]
|
||||
test_dirs.append(triple)
|
||||
elif opt == '--all':
|
||||
vers = 'all'
|
||||
for val in test_options_keys:
|
||||
test_dirs.append(test_options[val])
|
||||
|
||||
if do_coverage:
|
||||
os.environ['SPARK_PARSER_COVERAGE'] = (
|
||||
'/tmp/spark-grammar-%s.cover' % vers
|
||||
)
|
||||
|
||||
for src_dir, pattern, target_dir in test_dirs:
|
||||
if os.path.exists(src_dir):
|
||||
target_dir = os.path.join(target_base, target_dir)
|
||||
|
@@ -27,8 +27,6 @@ Step 2: Run the test:
|
||||
test_pythonlib.py --mylib --verify # decompile verify 'mylib'
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import getopt, os, py_compile, sys, shutil, tempfile, time
|
||||
|
||||
from uncompyle6 import PYTHON_VERSION
|
||||
@@ -127,8 +125,10 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
|
||||
if opts['do_compile']:
|
||||
compiled_version = opts['compiled_version']
|
||||
if compiled_version and PYTHON_VERSION != compiled_version:
|
||||
print("Not compiling: desired Python version is %s but we are running %s" %
|
||||
(compiled_version, PYTHON_VERSION), file=sys.stderr)
|
||||
sys.stderr.write("Not compiling: "
|
||||
"desired Python version is %s "
|
||||
"but we are running %s" %
|
||||
(compiled_version, PYTHON_VERSION))
|
||||
else:
|
||||
for root, dirs, basenames in os.walk(src_dir):
|
||||
file_matches(files, root, basenames, PY)
|
||||
@@ -146,8 +146,8 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
|
||||
file_matches(files, dirname, basenames, obj_patterns)
|
||||
|
||||
if not files:
|
||||
print("Didn't come up with any files to test! Try with --compile?",
|
||||
file=sys.stderr)
|
||||
sys.stderr.write("Didn't come up with any files to test! "
|
||||
"Try with --compile?")
|
||||
exit(1)
|
||||
|
||||
os.chdir(cwd)
|
||||
@@ -161,9 +161,9 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
print(time.ctime())
|
||||
print('Source directory: ', src_dir)
|
||||
print('Output directory: ', target_dir)
|
||||
print time.ctime()
|
||||
print 'Source directory: ', src_dir
|
||||
print 'Output directory: ', target_dir
|
||||
try:
|
||||
_, _, failed_files, failed_verify = \
|
||||
main(src_dir, target_dir, files, [],
|
||||
@@ -190,6 +190,7 @@ if __name__ == '__main__':
|
||||
test_options_keys.sort()
|
||||
opts, args = getopt.getopt(sys.argv[1:], '',
|
||||
['start-with=', 'verify', 'weak-verify', 'all', 'compile',
|
||||
'coverage',
|
||||
'no-rm'] \
|
||||
+ test_options_keys )
|
||||
if not opts: help()
|
||||
@@ -198,7 +199,8 @@ if __name__ == '__main__':
|
||||
'do_compile': False,
|
||||
'do_verify': False,
|
||||
'start_with': None,
|
||||
'rmtree' : True
|
||||
'rmtree' : True,
|
||||
'coverage' : False
|
||||
}
|
||||
|
||||
for opt, val in opts:
|
||||
@@ -217,24 +219,30 @@ if __name__ == '__main__':
|
||||
elif opt == '--all':
|
||||
for val in test_options_keys:
|
||||
test_dirs.append(test_options[val])
|
||||
elif opt == '--coverage':
|
||||
test_opts['coverage'] = True
|
||||
else:
|
||||
help()
|
||||
pass
|
||||
pass
|
||||
|
||||
if test_opts['coverage']:
|
||||
os.environ['SPARK_PARSER_COVERAGE'] = (
|
||||
'/tmp/spark-grammar-python-lib%s.cover' % test_dirs[0][-1]
|
||||
)
|
||||
|
||||
last_compile_version = None
|
||||
for src_dir, pattern, target_dir, compiled_version in test_dirs:
|
||||
if os.path.isdir(src_dir):
|
||||
checked_dirs.append([src_dir, pattern, target_dir])
|
||||
else:
|
||||
print("Can't find directory %s. Skipping" % src_dir,
|
||||
file=sys.stderr)
|
||||
sys.stderr.write("Can't find directory %s. Skipping" % src_dir)
|
||||
continue
|
||||
last_compile_version = compiled_version
|
||||
pass
|
||||
|
||||
if not checked_dirs:
|
||||
print("No directories found to check", file=sys.stderr)
|
||||
sys.stderr.write("No directories found to check\n")
|
||||
sys.exit(1)
|
||||
|
||||
test_opts['compiled_version'] = last_compile_version
|
||||
|
9
tox.ini
9
tox.ini
@@ -6,13 +6,14 @@ filename = *.py
|
||||
ignore = C901,E113,E121,E122,E123,E124,E125,E126,E127,E128,E129,E201,E202,E203,E221,E222,E225,E226,E241,E242,E251,E261,E271,E272,E302,E401,E501,F401,E701,E702
|
||||
|
||||
[tox]
|
||||
envlist = py26, py27, pypy
|
||||
envlist = py27, py34, pypy
|
||||
|
||||
[testenv]
|
||||
deps =
|
||||
requests>=0.8.8
|
||||
mock>=1.0.1
|
||||
commands = python -W always setup.py nosetests {posargs}
|
||||
hypothesis
|
||||
pytest
|
||||
flake8
|
||||
commands = python -W always make test {posargs}
|
||||
|
||||
[testenv:py27]
|
||||
deps =
|
||||
|
@@ -41,13 +41,18 @@ PYTHON_VERSION_STR = "%s.%s" % (sys.version_info[0], sys.version_info[1])
|
||||
|
||||
IS_PYPY = '__pypy__' in sys.builtin_module_names
|
||||
|
||||
sys.setrecursionlimit(5000)
|
||||
if hasattr(sys, 'setrecursionlimit'):
|
||||
# pyston doesn't have setrecursionlimit
|
||||
sys.setrecursionlimit(5000)
|
||||
|
||||
import uncompyle6.semantics.pysource
|
||||
import uncompyle6.semantics.fragments
|
||||
|
||||
# Export some functions
|
||||
from uncompyle6.main import uncompyle_file
|
||||
from uncompyle6.main import decompile_file
|
||||
|
||||
# For compaitility
|
||||
uncompyle_file = decompile_file
|
||||
|
||||
# Conventience functions so you can say:
|
||||
# from uncompyle6 import deparse_code
|
||||
|
@@ -3,7 +3,6 @@
|
||||
#
|
||||
# Copyright (c) 2015-2016 by Rocky Bernstein <rb@dustyfeet.com>
|
||||
#
|
||||
from __future__ import print_function
|
||||
import sys, os, getopt
|
||||
|
||||
from uncompyle6.disas import disassemble_file
|
||||
@@ -26,7 +25,7 @@ Options:
|
||||
-V | --version show version and stop
|
||||
-h | --help show this message
|
||||
|
||||
""".format(program)
|
||||
""" % (program, program)
|
||||
|
||||
PATTERNS = ('*.pyc', '*.pyo')
|
||||
|
||||
@@ -37,15 +36,15 @@ Type -h for for full help.""" % program
|
||||
native = True
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
print("No file(s) given", file=sys.stderr)
|
||||
print(Usage_short, file=sys.stderr)
|
||||
sys.stderr.write("No file(s) given\n")
|
||||
sys.stderr.write(Usage_short)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
opts, files = getopt.getopt(sys.argv[1:], 'hVU',
|
||||
['help', 'version', 'uncompyle6'])
|
||||
except getopt.GetoptError as e:
|
||||
print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr)
|
||||
except getopt.GetoptError(e):
|
||||
sys.stderr.write('%s: %s' % (os.path.basename(sys.argv[0]), e))
|
||||
sys.exit(-1)
|
||||
|
||||
for opt, val in opts:
|
||||
@@ -59,16 +58,14 @@ Type -h for for full help.""" % program
|
||||
native = False
|
||||
else:
|
||||
print(opt)
|
||||
print(Usage_short, file=sys.stderr)
|
||||
sys.stderr.write(Usage_short)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
for file in files:
|
||||
if os.path.exists(files[0]):
|
||||
disassemble_file(file, sys.stdout, native)
|
||||
else:
|
||||
print("Can't read %s - skipping" % files[0],
|
||||
file=sys.stderr)
|
||||
sys.stderr.write("Can't read %s - skipping\n" % files[0])
|
||||
pass
|
||||
pass
|
||||
return
|
||||
|
@@ -4,8 +4,7 @@
|
||||
# Copyright (c) 2015-2016 by Rocky Bernstein
|
||||
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
|
||||
#
|
||||
from __future__ import print_function
|
||||
import sys, os, getopt, tempfile, time
|
||||
import sys, os, getopt, time
|
||||
|
||||
program, ext = os.path.splitext(os.path.basename(__file__))
|
||||
|
||||
@@ -35,7 +34,8 @@ Options:
|
||||
-p <integer> use <integer> number of processes
|
||||
-r recurse directories looking for .pyc and .pyo files
|
||||
--verify compare generated source with input byte-code
|
||||
(requires -o)
|
||||
--linemaps generated line number correspondencies between byte-code
|
||||
and generated source output
|
||||
--help show this message
|
||||
|
||||
Debugging Options:
|
||||
@@ -64,11 +64,11 @@ def usage():
|
||||
|
||||
|
||||
def main_bin():
|
||||
if not (sys.version_info[0:2] in ((2, 6), (2, 7),
|
||||
if not (sys.version_info[0:2] in ((2, 4), (2, 5), (2, 6), (2, 7),
|
||||
(3, 2), (3, 3),
|
||||
(3, 4), (3, 5), (3, 6))):
|
||||
print('Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, or 3.6' % program,
|
||||
file=sys.stderr)
|
||||
sys.stderr.write('Error: %s requires Python 2.4 2.5 2.6, 2.7, '
|
||||
'3.2, 3.3, 3.4, 3.5, or 3.6' % program)
|
||||
sys.exit(-1)
|
||||
|
||||
do_verify = recurse_dirs = False
|
||||
@@ -81,10 +81,10 @@ def main_bin():
|
||||
|
||||
try:
|
||||
opts, files = getopt.getopt(sys.argv[1:], 'hagtdrVo:c:p:',
|
||||
'help asm grammar recurse timestamp tree verify version '
|
||||
'showgrammar'.split(' '))
|
||||
except getopt.GetoptError as e:
|
||||
print('%s: %s' % (os.path.basename(sys.argv[0]), e), file=sys.stderr)
|
||||
'help asm grammar linemaps recurse timestamp tree '
|
||||
'verify version showgrammar'.split(' '))
|
||||
except getopt.GetoptError(e):
|
||||
sys.stderr.write('%s: %s\n' % (os.path.basename(sys.argv[0]), e))
|
||||
sys.exit(-1)
|
||||
|
||||
options = {}
|
||||
@@ -97,6 +97,8 @@ def main_bin():
|
||||
sys.exit(0)
|
||||
elif opt == '--verify':
|
||||
options['do_verify'] = True
|
||||
elif opt == '--linemaps':
|
||||
options['do_linemaps'] = True
|
||||
elif opt in ('--asm', '-a'):
|
||||
options['showasm'] = 'after'
|
||||
options['do_verify'] = False
|
||||
@@ -116,7 +118,7 @@ def main_bin():
|
||||
elif opt in ('--recurse', '-r'):
|
||||
recurse_dirs = True
|
||||
else:
|
||||
print(opt, file=sys.stderr)
|
||||
sys.stderr.write(opt)
|
||||
usage()
|
||||
|
||||
# expand directory if specified
|
||||
@@ -139,18 +141,13 @@ def main_bin():
|
||||
if src_base:
|
||||
sb_len = len( os.path.join(src_base, '') )
|
||||
files = [f[sb_len:] for f in files]
|
||||
del sb_len
|
||||
|
||||
if not files:
|
||||
print("No files given", file=sys.stderr)
|
||||
sys.stderr.write("No files given\n")
|
||||
usage()
|
||||
|
||||
if outfile == '-':
|
||||
if 'do_verify' in options and options['do_verify'] and len(files) == 1:
|
||||
junk, outfile = tempfile.mkstemp(suffix=".pyc",
|
||||
prefix=files[0][0:-4]+'-')
|
||||
else:
|
||||
outfile = None # use stdout
|
||||
outfile = None # use stdout
|
||||
elif outfile and os.path.isdir(outfile):
|
||||
out_base = outfile; outfile = None
|
||||
elif outfile and len(files) > 1:
|
||||
@@ -226,7 +223,6 @@ def main_bin():
|
||||
except (KeyboardInterrupt, OSError):
|
||||
pass
|
||||
|
||||
|
||||
if timestamp:
|
||||
print(time.strftime(timestampfmt))
|
||||
|
||||
|
@@ -16,8 +16,6 @@ Second, we need structured instruction information for the
|
||||
want to run on Python 2.7.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
from collections import deque
|
||||
|
||||
@@ -37,10 +35,9 @@ def disco(version, co, out=None, is_pypy=False):
|
||||
|
||||
# store final output stream for case of error
|
||||
real_out = out or sys.stdout
|
||||
print('# Python %s' % version, file=real_out)
|
||||
real_out.write('# Python %s\n' % version)
|
||||
if co.co_filename:
|
||||
print('# Embedded file name: %s' % co.co_filename,
|
||||
file=real_out)
|
||||
real_out.write('# Embedded file name: %s\n' % co.co_filename)
|
||||
|
||||
scanner = get_scanner(version, is_pypy=is_pypy)
|
||||
|
||||
@@ -52,16 +49,15 @@ def disco_loop(disasm, queue, real_out):
|
||||
while len(queue) > 0:
|
||||
co = queue.popleft()
|
||||
if co.co_name != '<module>':
|
||||
print('\n# %s line %d of %s' %
|
||||
(co.co_name, co.co_firstlineno, co.co_filename),
|
||||
file=real_out)
|
||||
real_out.write('\n# %s line %d of %s\n' %
|
||||
(co.co_name, co.co_firstlineno, co.co_filename))
|
||||
tokens, customize = disasm(co)
|
||||
for t in tokens:
|
||||
if iscode(t.pattr):
|
||||
queue.append(t.pattr)
|
||||
elif iscode(t.attr):
|
||||
queue.append(t.attr)
|
||||
print(t, file=real_out)
|
||||
real_out.write(t)
|
||||
pass
|
||||
pass
|
||||
|
||||
|
61
uncompyle6/linenumbers.py
Normal file
61
uncompyle6/linenumbers.py
Normal file
@@ -0,0 +1,61 @@
|
||||
from collections import deque
|
||||
|
||||
from xdis.code import iscode
|
||||
from xdis.load import load_file, load_module
|
||||
from xdis.main import get_opcode
|
||||
from xdis.bytecode import Bytecode, findlinestarts, offset2line
|
||||
|
||||
def line_number_mapping(pyc_filename, src_filename):
|
||||
(version, timestamp, magic_int, code1, is_pypy,
|
||||
source_size) = load_module(pyc_filename)
|
||||
try:
|
||||
code2 = load_file(src_filename)
|
||||
except SyntaxError, e:
|
||||
return str(e)
|
||||
|
||||
queue = deque([code1, code2])
|
||||
|
||||
mappings = []
|
||||
|
||||
opc = get_opcode(version, is_pypy)
|
||||
number_loop(queue, mappings, opc)
|
||||
return sorted(mappings, key=lambda x: x[1])
|
||||
|
||||
|
||||
def number_loop(queue, mappings, opc):
|
||||
while len(queue) > 0:
|
||||
code1 = queue.popleft()
|
||||
code2 = queue.popleft()
|
||||
assert code1.co_name == code2.co_name
|
||||
linestarts_orig = findlinestarts(code1)
|
||||
linestarts_uncompiled = list(findlinestarts(code2))
|
||||
mappings += [[line, offset2line(offset, linestarts_uncompiled)] for offset, line in linestarts_orig]
|
||||
bytecode1 = Bytecode(code1, opc)
|
||||
bytecode2 = Bytecode(code2, opc)
|
||||
instr2s = bytecode2.get_instructions(code2)
|
||||
seen = set([code1.co_name])
|
||||
for instr in bytecode1.get_instructions(code1):
|
||||
next_code1 = None
|
||||
if iscode(instr.argval):
|
||||
next_code1 = instr.argval
|
||||
if next_code1:
|
||||
next_code2 = None
|
||||
while not next_code2:
|
||||
try:
|
||||
instr2 = next(instr2s)
|
||||
if iscode(instr2.argval):
|
||||
next_code2 = instr2.argval
|
||||
pass
|
||||
except StopIteration:
|
||||
break
|
||||
pass
|
||||
if next_code2:
|
||||
assert next_code1.co_name == next_code2.co_name
|
||||
if next_code1.co_name not in seen:
|
||||
seen.add(next_code1.co_name)
|
||||
queue.append(next_code1)
|
||||
queue.append(next_code2)
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
pass
|
@@ -1,5 +1,4 @@
|
||||
from __future__ import print_function
|
||||
import datetime, os, sys
|
||||
import datetime, os, subprocess, sys, tempfile
|
||||
|
||||
from uncompyle6 import verify, IS_PYPY
|
||||
from xdis.code import iscode
|
||||
@@ -7,10 +6,11 @@ from uncompyle6.disas import check_object_path
|
||||
from uncompyle6.semantics import pysource
|
||||
from uncompyle6.parser import ParserError
|
||||
from uncompyle6.version import VERSION
|
||||
from uncompyle6.linenumbers import line_number_mapping
|
||||
|
||||
from xdis.load import load_module
|
||||
|
||||
def uncompyle(
|
||||
def decompile(
|
||||
bytecode_version, co, out=None, showasm=None, showast=False,
|
||||
timestamp=None, showgrammar=False, code_objects={},
|
||||
source_size=None, is_pypy=False, magic_int=None):
|
||||
@@ -21,35 +21,41 @@ def uncompyle(
|
||||
|
||||
# store final output stream for case of error
|
||||
real_out = out or sys.stdout
|
||||
co_pypy_str = 'PyPy ' if is_pypy else ''
|
||||
run_pypy_str = 'PyPy ' if IS_PYPY else ''
|
||||
print('# uncompyle6 version %s\n'
|
||||
'# %sPython bytecode %s%s\n# Decompiled from: %sPython %s' %
|
||||
if is_pypy:
|
||||
co_pypy_str = 'PyPy '
|
||||
else:
|
||||
co_pypy_str = ''
|
||||
|
||||
if IS_PYPY:
|
||||
run_pypy_str = 'PyPy '
|
||||
else:
|
||||
run_pypy_str = ''
|
||||
|
||||
if magic_int:
|
||||
m = str(magic_int)
|
||||
else:
|
||||
m = ""
|
||||
real_out.write('# uncompyle6 version %s\n'
|
||||
'# %sPython bytecode %s%s\n# Decompiled from: %sPython %s\n' %
|
||||
(VERSION, co_pypy_str, bytecode_version,
|
||||
" (%d)" % magic_int if magic_int else "",
|
||||
run_pypy_str, '\n# '.join(sys.version.split('\n'))),
|
||||
file=real_out)
|
||||
" (%s)" % m, run_pypy_str,
|
||||
'\n# '.join(sys.version.split('\n'))))
|
||||
if co.co_filename:
|
||||
print('# Embedded file name: %s' % co.co_filename,
|
||||
file=real_out)
|
||||
real_out.write('# Embedded file name: %s\n' % co.co_filename)
|
||||
if timestamp:
|
||||
print('# Compiled at: %s' % datetime.datetime.fromtimestamp(timestamp),
|
||||
file=real_out)
|
||||
real_out.write('# Compiled at: %s\n' %
|
||||
datetime.datetime.fromtimestamp(timestamp))
|
||||
if source_size:
|
||||
print('# Size of source mod 2**32: %d bytes' % source_size,
|
||||
file=real_out)
|
||||
real_out.write('# Size of source mod 2**32: %d bytes\n' % source_size)
|
||||
|
||||
try:
|
||||
pysource.deparse_code(bytecode_version, co, out, showasm, showast,
|
||||
showgrammar, code_objects=code_objects,
|
||||
is_pypy=is_pypy)
|
||||
except pysource.SourceWalkerError as e:
|
||||
# deparsing failed
|
||||
raise pysource.SourceWalkerError(str(e))
|
||||
pysource.deparse_code(bytecode_version, co, out, showasm, showast,
|
||||
showgrammar, code_objects=code_objects,
|
||||
is_pypy=is_pypy)
|
||||
|
||||
# For compatiblity
|
||||
uncompyle = decompile
|
||||
|
||||
|
||||
def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
|
||||
def decompile_file(filename, outstream=None, showasm=None, showast=False,
|
||||
showgrammar=False):
|
||||
"""
|
||||
decompile Python byte-code file (.pyc)
|
||||
@@ -60,23 +66,27 @@ def uncompyle_file(filename, outstream=None, showasm=None, showast=False,
|
||||
(version, timestamp, magic_int, co, is_pypy,
|
||||
source_size) = load_module(filename, code_objects)
|
||||
|
||||
|
||||
if type(co) == list:
|
||||
for con in co:
|
||||
uncompyle(version, con, outstream, showasm, showast,
|
||||
decompile(version, con, outstream, showasm, showast,
|
||||
timestamp, showgrammar, code_objects=code_objects,
|
||||
is_pypy=is_pypy, magic_int=magic_int)
|
||||
else:
|
||||
uncompyle(version, co, outstream, showasm, showast,
|
||||
decompile(version, co, outstream, showasm, showast,
|
||||
timestamp, showgrammar,
|
||||
code_objects=code_objects, source_size=source_size,
|
||||
is_pypy=is_pypy, magic_int=magic_int)
|
||||
co = None
|
||||
|
||||
# For compatiblity
|
||||
uncompyle_file = decompile_file
|
||||
|
||||
|
||||
# FIXME: combine into an options parameter
|
||||
def main(in_base, out_base, files, codes, outfile=None,
|
||||
showasm=None, showast=False, do_verify=False,
|
||||
showgrammar=False, raise_on_error=False):
|
||||
showgrammar=False, raise_on_error=False,
|
||||
do_linemaps=False):
|
||||
"""
|
||||
in_base base directory for input files
|
||||
out_base base directory for output files (ignored when
|
||||
@@ -99,15 +109,8 @@ def main(in_base, out_base, files, codes, outfile=None,
|
||||
pass
|
||||
return open(outfile, 'w')
|
||||
|
||||
of = outfile
|
||||
tot_files = okay_files = failed_files = verify_failed_files = 0
|
||||
|
||||
# for code in codes:
|
||||
# version = sys.version[:3] # "2.5"
|
||||
# with open(code, "r") as f:
|
||||
# co = compile(f.read(), "", "exec")
|
||||
# uncompyle(sys.version[:3], co, sys.stdout, showasm=showasm, showast=showast)
|
||||
|
||||
for filename in files:
|
||||
infile = os.path.join(in_base, filename)
|
||||
if not os.path.exists(infile):
|
||||
@@ -117,10 +120,25 @@ def main(in_base, out_base, files, codes, outfile=None,
|
||||
|
||||
# print (infile, file=sys.stderr)
|
||||
|
||||
if of: # outfile was given as parameter
|
||||
if outfile: # outfile was given as parameter
|
||||
outstream = _get_outstream(outfile)
|
||||
elif out_base is None:
|
||||
outstream = sys.stdout
|
||||
if do_linemaps or do_verify:
|
||||
prefix = os.path.basename(filename)
|
||||
if prefix.endswith('.py'):
|
||||
prefix = prefix[:-len('.py')]
|
||||
junk, outfile = tempfile.mkstemp(suffix=".py",
|
||||
prefix=prefix)
|
||||
# Unbuffer output if possible
|
||||
if sys.stdout.isatty():
|
||||
buffering = -1
|
||||
else:
|
||||
buffering = 0
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering)
|
||||
tee = subprocess.Popen(["tee", outfile], stdin=subprocess.PIPE)
|
||||
os.dup2(tee.stdin.fileno(), sys.stdout.fileno())
|
||||
os.dup2(tee.stdin.fileno(), sys.stderr.fileno())
|
||||
else:
|
||||
if filename.endswith('.pyc'):
|
||||
outfile = os.path.join(out_base, filename[0:-1])
|
||||
@@ -131,15 +149,17 @@ def main(in_base, out_base, files, codes, outfile=None,
|
||||
|
||||
# Try to uncompile the input file
|
||||
try:
|
||||
uncompyle_file(infile, outstream, showasm, showast, showgrammar)
|
||||
decompile_file(infile, outstream, showasm, showast, showgrammar)
|
||||
tot_files += 1
|
||||
except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
|
||||
sys.stderr.write("\n# file %s\n# %s" % (infile, e))
|
||||
except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError):
|
||||
sys.stdout.write("\n")
|
||||
sys.stderr.write("# file %s\n" % (infile))
|
||||
failed_files += 1
|
||||
except KeyboardInterrupt:
|
||||
if outfile:
|
||||
outstream.close()
|
||||
os.remove(outfile)
|
||||
sys.stdout.write("\n")
|
||||
sys.stderr.write("\nLast file: %s " % (infile))
|
||||
raise
|
||||
# except:
|
||||
@@ -152,39 +172,50 @@ def main(in_base, out_base, files, codes, outfile=None,
|
||||
# sys.stderr.write("\n# Can't uncompile %s\n" % infile)
|
||||
else: # uncompile successful
|
||||
if outfile:
|
||||
if do_linemaps:
|
||||
mapping = line_number_mapping(infile, outfile)
|
||||
outstream.write("\n\n## Line number correspondences\n")
|
||||
import pprint
|
||||
s = pprint.pformat(mapping, indent=2, width=80)
|
||||
s2 = '##' + '\n##'.join(s.split("\n")) + "\n"
|
||||
outstream.write(s2)
|
||||
outstream.close()
|
||||
|
||||
if do_verify:
|
||||
weak_verify = do_verify == 'weak'
|
||||
try:
|
||||
msg = verify.compare_code_with_srcfile(infile, outfile, weak_verify=weak_verify)
|
||||
if not outfile:
|
||||
if not msg:
|
||||
print('\n# okay decompiling %s' % infile)
|
||||
print '\n# okay decompiling %s' % infile
|
||||
okay_files += 1
|
||||
else:
|
||||
print('\n# %s\n\t%s', infile, msg)
|
||||
except verify.VerifyCmpError as e:
|
||||
print '\n# %s\n\t%s', infile, msg
|
||||
except verify.VerifyCmpError, e:
|
||||
print(e)
|
||||
verify_failed_files += 1
|
||||
os.rename(outfile, outfile + '_unverified')
|
||||
sys.stderr.write("### Error Verifying %s\n" % filename)
|
||||
sys.stderr.write(str(e) + "\n")
|
||||
if not outfile:
|
||||
print("### Error Verifiying %s" % filename, file=sys.stderr)
|
||||
print(e, file=sys.stderr)
|
||||
sys.stder.write("### Error Verifiying %s" %
|
||||
filename)
|
||||
sys.stderr.write(e)
|
||||
if raise_on_error:
|
||||
raise
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
elif do_verify:
|
||||
print("\n### uncompile successful, but no file to compare against",
|
||||
file=sys.stderr)
|
||||
sys.stderr.write("\n### uncompile successful, "
|
||||
"but no file to compare against")
|
||||
pass
|
||||
else:
|
||||
okay_files += 1
|
||||
if not outfile:
|
||||
mess = '\n# okay decompiling'
|
||||
# mem_usage = __memUsage()
|
||||
print(mess, infile)
|
||||
print mess, infile
|
||||
if outfile:
|
||||
sys.stdout.write("%s\r" %
|
||||
status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files))
|
||||
@@ -211,11 +242,11 @@ def status_msg(do_verify, tot_files, okay_files, failed_files,
|
||||
verify_failed_files):
|
||||
if tot_files == 1:
|
||||
if failed_files:
|
||||
return "decompile failed"
|
||||
return "\n# decompile failed"
|
||||
elif verify_failed_files:
|
||||
return "decompile verify failed"
|
||||
return "\n# decompile verify failed"
|
||||
else:
|
||||
return "Successfully decompiled file"
|
||||
return "\n# Successfully decompiled file"
|
||||
pass
|
||||
pass
|
||||
mess = "decompiled %i files: %i okay, %i failed" % (tot_files, okay_files, failed_files)
|
||||
|
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2015-2016 Rocky Bernstein
|
||||
# Copyright (c) 2015-2017 Rocky Bernstein
|
||||
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
|
||||
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
|
||||
# Copyright (c) 1999 John Aycock
|
||||
@@ -6,8 +6,6 @@
|
||||
Common uncompyle parser routines.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
|
||||
from xdis.code import iscode
|
||||
@@ -28,6 +26,16 @@ nop_func = lambda self, args: None
|
||||
|
||||
class PythonParser(GenericASTBuilder):
|
||||
|
||||
def __init__(self, AST, start, debug):
|
||||
super(PythonParser, self).__init__(AST, start, debug)
|
||||
self.collect = [
|
||||
'stmts', 'except_stmts', '_stmts',
|
||||
'exprlist', 'kvlist', 'kwargs', 'come_froms',
|
||||
# Python < 3
|
||||
'print_items',
|
||||
# PyPy:
|
||||
'kvlist_n']
|
||||
|
||||
def add_unique_rule(self, rule, opname, count, customize):
|
||||
"""Add rule to grammar, but only if it hasn't been added previously
|
||||
opname and count are used in the customize() semantic the actions
|
||||
@@ -69,6 +77,36 @@ class PythonParser(GenericASTBuilder):
|
||||
for i in dir(self):
|
||||
setattr(self, i, None)
|
||||
|
||||
def debug_reduce(self, rule, tokens, parent, i):
|
||||
"""Customized format and print for our kind of tokens
|
||||
which gets called in debugging grammar reduce rules
|
||||
"""
|
||||
def fix(c):
|
||||
s = str(c)
|
||||
i = s.find('_')
|
||||
if i == -1:
|
||||
return s
|
||||
else:
|
||||
return s[:i]
|
||||
|
||||
prefix = ''
|
||||
if parent and tokens:
|
||||
p_token = tokens[parent]
|
||||
if hasattr(p_token, 'linestart') and p_token.linestart:
|
||||
prefix = 'L.%3d: ' % p_token.linestart
|
||||
else:
|
||||
prefix = ' '
|
||||
if hasattr(p_token, 'offset'):
|
||||
prefix += "%3s" % fix(p_token.offset)
|
||||
if len(rule[1]) > 1:
|
||||
prefix += '-%-3s ' % fix(tokens[i-1].offset)
|
||||
else:
|
||||
prefix += ' '
|
||||
else:
|
||||
prefix = ' '
|
||||
|
||||
print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1])))
|
||||
|
||||
def error(self, instructions, index):
|
||||
# Find the last line boundary
|
||||
for start in range(index, -1, -1):
|
||||
@@ -80,7 +118,10 @@ class PythonParser(GenericASTBuilder):
|
||||
err_token = instructions[index]
|
||||
print("Instruction context:")
|
||||
for i in range(start, finish):
|
||||
indent = ' ' if i != index else '-> '
|
||||
if i != index:
|
||||
indent = ' '
|
||||
else:
|
||||
indent = '-> '
|
||||
print("%s%s" % (indent, instructions[i]))
|
||||
raise ParserError(err_token, err_token.offset)
|
||||
|
||||
@@ -88,11 +129,7 @@ class PythonParser(GenericASTBuilder):
|
||||
return token.type
|
||||
|
||||
def nonterminal(self, nt, args):
|
||||
collect = ('stmts', 'exprlist', 'kvlist', '_stmts', 'print_items', 'kwargs',
|
||||
# PYPY:
|
||||
'kvlist_n')
|
||||
|
||||
if nt in collect and len(args) > 1:
|
||||
if nt in self.collect and len(args) > 1:
|
||||
#
|
||||
# Collect iterated thingies together. That is rather than
|
||||
# stmts -> stmts stmt -> stmts stmt -> ...
|
||||
@@ -117,9 +154,9 @@ class PythonParser(GenericASTBuilder):
|
||||
# print >> sys.stderr, 'resolve', str(list)
|
||||
return GenericASTBuilder.resolve(self, list)
|
||||
|
||||
##############################################
|
||||
## Common Python 2 and Python 3 grammar rules
|
||||
##############################################
|
||||
###############################################
|
||||
# Common Python 2 and Python 3 grammar rules #
|
||||
###############################################
|
||||
def p_start(self, args):
|
||||
'''
|
||||
# The start or goal symbol
|
||||
@@ -138,8 +175,7 @@ class PythonParser(GenericASTBuilder):
|
||||
"""
|
||||
passstmt ::=
|
||||
|
||||
_stmts ::= _stmts stmt
|
||||
_stmts ::= stmt
|
||||
_stmts ::= stmt+
|
||||
|
||||
# statements with continue
|
||||
c_stmts ::= _stmts
|
||||
@@ -246,13 +282,11 @@ class PythonParser(GenericASTBuilder):
|
||||
|
||||
# Zero or more COME_FROMs
|
||||
# loops can have this
|
||||
_come_from ::= _come_from COME_FROM
|
||||
_come_from ::=
|
||||
_come_from ::= COME_FROM*
|
||||
|
||||
# Zero or one COME_FROM
|
||||
# And/or expressions have this
|
||||
come_from_opt ::= COME_FROM
|
||||
come_from_opt ::=
|
||||
come_from_opt ::= COME_FROM?
|
||||
"""
|
||||
|
||||
def p_dictcomp(self, args):
|
||||
@@ -425,7 +459,6 @@ class PythonParser(GenericASTBuilder):
|
||||
expr ::= unary_not
|
||||
expr ::= binary_subscr
|
||||
expr ::= binary_subscr2
|
||||
expr ::= load_attr
|
||||
expr ::= get_iter
|
||||
expr ::= buildslice2
|
||||
expr ::= buildslice3
|
||||
@@ -467,6 +500,8 @@ class PythonParser(GenericASTBuilder):
|
||||
_mklambda ::= load_closure mklambda
|
||||
_mklambda ::= mklambda
|
||||
|
||||
# "and" where the first part of the and is true,
|
||||
# so there is only the 2nd part to evaluate
|
||||
and2 ::= _jump jmp_false COME_FROM expr COME_FROM
|
||||
|
||||
expr ::= conditional
|
||||
@@ -556,7 +591,7 @@ def parse(p, tokens, customize):
|
||||
|
||||
|
||||
def get_python_parser(
|
||||
version, debug_parser={}, compile_mode='exec',
|
||||
version, debug_parser=PARSER_DEFAULT_DEBUG, compile_mode='exec',
|
||||
is_pypy = False):
|
||||
"""Returns parser object for Python version 2 or 3, 3.2, 3.5on,
|
||||
etc., depending on the parameters passed. *compile_mode* is either
|
||||
@@ -710,8 +745,8 @@ def python_parser(version, co, out=sys.stdout, showasm=False,
|
||||
maybe_show_asm(showasm, tokens)
|
||||
|
||||
# For heavy grammar debugging
|
||||
parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
|
||||
'showstack': 'full'}
|
||||
# parser_debug = {'rules': True, 'transition': True, 'reduce' : True,
|
||||
# 'showstack': 'full'}
|
||||
p = get_python_parser(version, parser_debug)
|
||||
return parse(p, tokens, customize)
|
||||
|
||||
|
@@ -28,9 +28,9 @@ class AST(spark_AST):
|
||||
i = 0
|
||||
for node in self:
|
||||
if hasattr(node, '__repr1__'):
|
||||
if enumerate_children:
|
||||
if enumerate_children:
|
||||
child = node.__repr1__(indent, i)
|
||||
else:
|
||||
else:
|
||||
child = node.__repr1__(indent, None)
|
||||
else:
|
||||
inst = node.format(line_prefix='L.')
|
||||
|
@@ -9,11 +9,9 @@ e.g. 5, myvariable, "for", etc. they are CPython Bytecode tokens,
|
||||
e.g. "LOAD_CONST 5", "STORE NAME myvariable", "SETUP_LOOP", etc.
|
||||
|
||||
If we succeed in creating a parse tree, then we have a Python program
|
||||
that a later phase can tern into a sequence of ASCII text.
|
||||
that a later phase can turn into a sequence of ASCII text.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
|
||||
from uncompyle6.parsers.astnode import AST
|
||||
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
|
||||
@@ -25,20 +23,18 @@ class Python2Parser(PythonParser):
|
||||
self.new_rules = set()
|
||||
|
||||
def p_print2(self, args):
|
||||
'''
|
||||
"""
|
||||
stmt ::= print_items_stmt
|
||||
stmt ::= print_nl
|
||||
stmt ::= print_items_nl_stmt
|
||||
|
||||
print_items_stmt ::= expr PRINT_ITEM print_items_opt
|
||||
print_items_nl_stmt ::= expr PRINT_ITEM print_items_opt PRINT_NEWLINE_CONT
|
||||
print_items_opt ::= print_items
|
||||
print_items_opt ::=
|
||||
print_items ::= print_items print_item
|
||||
print_items ::= print_item
|
||||
print_item ::= expr PRINT_ITEM_CONT
|
||||
print_nl ::= PRINT_NEWLINE
|
||||
'''
|
||||
print_items_opt ::= print_items?
|
||||
print_items ::= print_item+
|
||||
print_item ::= expr PRINT_ITEM_CONT
|
||||
print_nl ::= PRINT_NEWLINE
|
||||
"""
|
||||
|
||||
def p_stmt2(self, args):
|
||||
"""
|
||||
@@ -76,8 +72,6 @@ class Python2Parser(PythonParser):
|
||||
return_if_stmts ::= _stmts return_if_stmt
|
||||
return_if_stmt ::= ret_expr RETURN_END_IF
|
||||
|
||||
stmt ::= importstmt
|
||||
|
||||
stmt ::= break_stmt
|
||||
break_stmt ::= BREAK_LOOP
|
||||
|
||||
@@ -130,6 +124,7 @@ class Python2Parser(PythonParser):
|
||||
assert_expr_and ::= assert_expr jmp_false expr
|
||||
|
||||
ifstmt ::= testexpr _ifstmts_jump
|
||||
ifstmt ::= testexpr return_if_stmts COME_FROM
|
||||
|
||||
testexpr ::= testfalse
|
||||
testexpr ::= testtrue
|
||||
@@ -148,6 +143,8 @@ class Python2Parser(PythonParser):
|
||||
|
||||
ifelsestmtr ::= testexpr return_if_stmts return_stmts
|
||||
|
||||
ifelsestmtr ::= testexpr return_if_stmts COME_FROM return_stmts
|
||||
|
||||
ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
|
||||
|
||||
|
||||
@@ -171,8 +168,7 @@ class Python2Parser(PythonParser):
|
||||
try_middle ::= jmp_abs COME_FROM except_stmts
|
||||
END_FINALLY
|
||||
|
||||
except_stmts ::= except_stmts except_stmt
|
||||
except_stmts ::= except_stmt
|
||||
except_stmts ::= except_stmt+
|
||||
|
||||
except_stmt ::= except_cond1 except_suite
|
||||
except_stmt ::= except
|
||||
@@ -210,14 +206,6 @@ class Python2Parser(PythonParser):
|
||||
and ::= expr jmp_false expr come_from_opt
|
||||
or ::= expr jmp_true expr come_from_opt
|
||||
|
||||
slice0 ::= expr SLICE+0
|
||||
slice0 ::= expr DUP_TOP SLICE+0
|
||||
slice1 ::= expr expr SLICE+1
|
||||
slice1 ::= expr expr DUP_TOPX_2 SLICE+1
|
||||
slice2 ::= expr expr SLICE+2
|
||||
slice2 ::= expr expr DUP_TOPX_2 SLICE+2
|
||||
slice3 ::= expr expr expr SLICE+3
|
||||
slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
|
||||
unary_convert ::= expr UNARY_CONVERT
|
||||
|
||||
# In Python 3, DUP_TOPX_2 is DUP_TOP_TWO
|
||||
@@ -248,11 +236,10 @@ class Python2Parser(PythonParser):
|
||||
"""
|
||||
inplace_op ::= INPLACE_DIVIDE
|
||||
binary_op ::= BINARY_DIVIDE
|
||||
binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
|
||||
"""
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
'''
|
||||
"""
|
||||
Special handling for opcodes such as those that take a variable number
|
||||
of arguments -- we add a new rule for each:
|
||||
|
||||
@@ -271,7 +258,7 @@ class Python2Parser(PythonParser):
|
||||
expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
|
||||
|
||||
PyPy adds custom rules here as well
|
||||
'''
|
||||
"""
|
||||
for opname, v in list(customize.items()):
|
||||
opname_base = opname[:opname.rfind('_')]
|
||||
if opname == 'PyPy':
|
||||
@@ -286,7 +273,7 @@ class Python2Parser(PythonParser):
|
||||
continue
|
||||
elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
|
||||
thousands = (v//1024)
|
||||
thirty32s = ((v//32)%32)
|
||||
thirty32s = ((v//32) % 32)
|
||||
if thirty32s > 0:
|
||||
rule = "expr32 ::=%s" % (' expr' * 32)
|
||||
self.add_unique_rule(rule, opname_base, v, customize)
|
||||
@@ -296,7 +283,7 @@ class Python2Parser(PythonParser):
|
||||
opname_base, v, customize)
|
||||
self.seen1024 = True
|
||||
rule = ('build_list ::= ' + 'expr1024 '*thousands +
|
||||
'expr32 '*thirty32s + 'expr '*(v%32) + opname)
|
||||
'expr32 '*thirty32s + 'expr '*(v % 32) + opname)
|
||||
elif opname == 'LOOKUP_METHOD':
|
||||
# A PyPy speciality - DRY with parse3
|
||||
self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD",
|
||||
@@ -347,7 +334,7 @@ class Python2Parser(PythonParser):
|
||||
# always be the case.
|
||||
self.add_unique_rules([
|
||||
"stmt ::= tryfinallystmt_pypy",
|
||||
"tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM "
|
||||
"tryfinallystmt_pypy ::= SETUP_FINALLY suite_stmts_opt COME_FROM_FINALLY "
|
||||
"suite_stmts_opt END_FINALLY"
|
||||
], customize)
|
||||
continue
|
||||
@@ -400,6 +387,26 @@ class Python2Parser(PythonParser):
|
||||
else:
|
||||
raise Exception('unknown customize token %s' % opname)
|
||||
self.add_unique_rule(rule, opname_base, v, customize)
|
||||
pass
|
||||
self.check_reduce['augassign1'] = 'AST'
|
||||
self.check_reduce['augassign2'] = 'AST'
|
||||
self.check_reduce['_stmts'] = 'AST'
|
||||
return
|
||||
|
||||
def reduce_is_invalid(self, rule, ast, tokens, first, last):
|
||||
lhs = rule[0]
|
||||
if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
|
||||
return True
|
||||
elif lhs == '_stmts':
|
||||
for i, stmt in enumerate(ast):
|
||||
if stmt == '_stmts':
|
||||
stmt = stmt[0]
|
||||
assert stmt == 'stmt'
|
||||
if stmt[0] == 'return_stmt':
|
||||
return i+1 != len(ast)
|
||||
pass
|
||||
return False
|
||||
return False
|
||||
|
||||
class Python2ParserSingle(Python2Parser, PythonParserSingle):
|
||||
pass
|
||||
|
@@ -18,6 +18,9 @@ class Python23Parser(Python24Parser):
|
||||
# of Python
|
||||
_while1test ::= SETUP_LOOP JUMP_FORWARD JUMP_IF_FALSE POP_TOP COME_FROM
|
||||
|
||||
while1stmt ::= _while1test l_stmts_opt JUMP_BACK
|
||||
POP_TOP POP_BLOCK COME_FROM
|
||||
|
||||
while1stmt ::= _while1test l_stmts_opt JUMP_BACK
|
||||
COME_FROM POP_TOP POP_BLOCK COME_FROM
|
||||
|
||||
|
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2016 Rocky Bernstein
|
||||
# Copyright (c) 2016-2017 Rocky Bernstein
|
||||
"""
|
||||
spark grammar differences over Python2.6 for Python 2.5.
|
||||
"""
|
||||
@@ -13,27 +13,46 @@ class Python25Parser(Python26Parser):
|
||||
self.customized = {}
|
||||
|
||||
def p_misc25(self, args):
|
||||
'''
|
||||
"""
|
||||
# If "return_if_stmt" is in a loop, a JUMP_BACK can be emitted. In 2.6 the
|
||||
# JUMP_BACK doesn't appear
|
||||
|
||||
return_if_stmt ::= ret_expr RETURN_END_IF JUMP_BACK
|
||||
|
||||
# Python 2.6 uses ROT_TWO instead of the STORE_xxx
|
||||
# withas is allowed as a "from future" in 2.5
|
||||
setupwithas ::= DUP_TOP LOAD_ATTR store LOAD_ATTR CALL_FUNCTION_0
|
||||
setup_finally
|
||||
|
||||
store ::= STORE_FAST
|
||||
store ::= STORE_NAME
|
||||
|
||||
# Python 2.6 omits ths LOAD_FAST DELETE_FAST below
|
||||
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
|
||||
try_middle else_suite COME_FROM
|
||||
|
||||
# Python 2.6 omits the LOAD_FAST DELETE_FAST below
|
||||
# withas is allowed as a "from future" in 2.5
|
||||
withasstmt ::= expr setupwithas designator suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM
|
||||
with_cleanup
|
||||
|
||||
with_cleanup ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
|
||||
with_cleanup ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY
|
||||
'''
|
||||
"""
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
super(Python25Parser, self).add_custom_rules(tokens, customize)
|
||||
if self.version == 2.5:
|
||||
self.check_reduce['tryelsestmt'] = 'tokens'
|
||||
|
||||
def reduce_is_invalid(self, rule, ast, tokens, first, last):
|
||||
invalid = super(Python25Parser,
|
||||
self).reduce_is_invalid(rule, ast,
|
||||
tokens, first, last)
|
||||
if invalid:
|
||||
return invalid
|
||||
return False
|
||||
|
||||
|
||||
class Python25ParserSingle(Python26Parser, PythonParserSingle):
|
||||
pass
|
||||
|
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2016 Rocky Bernstein
|
||||
# Copyright (c) 2017 Rocky Bernstein
|
||||
"""
|
||||
spark grammar differences over Python2 for Python 2.6.
|
||||
"""
|
||||
@@ -13,7 +13,6 @@ class Python26Parser(Python2Parser):
|
||||
super(Python26Parser, self).__init__(debug_parser)
|
||||
self.customized = {}
|
||||
|
||||
|
||||
def p_try_except26(self, args):
|
||||
"""
|
||||
except_stmt ::= except_cond3 except_suite
|
||||
@@ -23,36 +22,39 @@ class Python26Parser(Python2Parser):
|
||||
JUMP_IF_FALSE POP_TOP POP_TOP designator POP_TOP
|
||||
|
||||
try_middle ::= JUMP_FORWARD COME_FROM except_stmts
|
||||
come_from_pop END_FINALLY COME_FROM
|
||||
come_from_pop END_FINALLY come_froms
|
||||
|
||||
try_middle ::= JUMP_FORWARD COME_FROM except_stmts END_FINALLY
|
||||
come_froms
|
||||
|
||||
try_middle ::= jmp_abs COME_FROM except_stmts
|
||||
POP_TOP END_FINALLY
|
||||
|
||||
try_middle ::= jmp_abs COME_FROM except_stmts
|
||||
come_from_pop END_FINALLY
|
||||
|
||||
trystmt ::= SETUP_EXCEPT suite_stmts_opt come_from_pop
|
||||
try_middle
|
||||
END_FINALLY JUMP_FORWARD
|
||||
|
||||
# Sometimes we don't put in COME_FROM to the next statement
|
||||
# like we do in 2.7. Perhaps we should?
|
||||
trystmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
|
||||
try_middle
|
||||
|
||||
trystmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
|
||||
try_middle come_froms
|
||||
|
||||
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
|
||||
try_middle else_suite come_froms
|
||||
try_middle else_suite COME_FROM
|
||||
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
|
||||
|
||||
except_suite ::= c_stmts_opt JUMP_FORWARD come_from_pop
|
||||
except_suite ::= c_stmts_opt JUMP_FORWARD POP_TOP
|
||||
except_suite ::= c_stmts_opt jmp_abs come_from_pop
|
||||
|
||||
# Python 3 also has this.
|
||||
come_froms ::= come_froms COME_FROM
|
||||
come_froms ::= COME_FROM
|
||||
|
||||
# This is what happens after a jump where
|
||||
# we start a new block. For reasons I don't fully
|
||||
# understand, there is also a value on the top of the stack
|
||||
come_from_pop ::= COME_FROM POP_TOP
|
||||
come_froms_pop ::= come_froms POP_TOP
|
||||
|
||||
"""
|
||||
@@ -70,14 +72,15 @@ class Python26Parser(Python2Parser):
|
||||
jmp_true ::= JUMP_IF_TRUE POP_TOP
|
||||
jmp_false ::= JUMP_IF_FALSE POP_TOP
|
||||
|
||||
jf_pop ::= JUMP_FORWARD come_from_pop
|
||||
jf_pop ::= JUMP_ABSOLUTE come_from_pop
|
||||
jb_pop ::= JUMP_BACK come_from_pop
|
||||
jf_pop ::= JUMP_FORWARD POP_TOP
|
||||
jf_pop ::= JUMP_ABSOLUTE POP_TOP
|
||||
jb_pop ::= JUMP_BACK POP_TOP
|
||||
|
||||
jb_cont ::= JUMP_BACK
|
||||
jb_cont ::= CONTINUE
|
||||
|
||||
jb_cf_pop ::= JUMP_BACK come_froms POP_TOP
|
||||
jb_cf_pop ::= JUMP_BACK POP_TOP
|
||||
ja_cf_pop ::= JUMP_ABSOLUTE come_froms POP_TOP
|
||||
jf_cf_pop ::= JUMP_FORWARD come_froms POP_TOP
|
||||
|
||||
@@ -85,13 +88,12 @@ class Python26Parser(Python2Parser):
|
||||
jb_bp_come_from ::= JUMP_BACK bp_come_from
|
||||
|
||||
_ifstmts_jump ::= c_stmts_opt jf_pop COME_FROM
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM come_from_pop
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD come_froms POP_TOP COME_FROM
|
||||
|
||||
# This is what happens after a jump where
|
||||
# we start a new block. For reasons I don't fully
|
||||
# understand, there is also a value on the top of the stack
|
||||
come_from_pop ::= COME_FROM POP_TOP
|
||||
come_froms_pop ::= come_froms POP_TOP
|
||||
|
||||
"""
|
||||
@@ -108,16 +110,10 @@ class Python26Parser(Python2Parser):
|
||||
|
||||
break_stmt ::= BREAK_LOOP JUMP_BACK
|
||||
|
||||
# Semantic actions want the else to be at position 3
|
||||
ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
|
||||
ifelsestmt ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
|
||||
|
||||
# Semantic actions want else_suitel to be at index 3
|
||||
ifelsestmtl ::= testexpr c_stmts_opt jb_cf_pop else_suitel
|
||||
ifelsestmtc ::= testexpr c_stmts_opt ja_cf_pop else_suitec
|
||||
|
||||
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
|
||||
|
||||
# Semantic actions want suite_stmts_opt to be at index 3
|
||||
withstmt ::= expr setupwith SETUP_FINALLY suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
|
||||
@@ -145,15 +141,40 @@ class Python26Parser(Python2Parser):
|
||||
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt jb_pop POP_BLOCK
|
||||
else_suite COME_FROM
|
||||
|
||||
return_stmt ::= ret_expr RETURN_END_IF come_from_pop
|
||||
return_stmt ::= ret_expr RETURN_VALUE come_from_pop
|
||||
return_if_stmt ::= ret_expr RETURN_END_IF come_from_pop
|
||||
return_stmt ::= ret_expr RETURN_END_IF POP_TOP
|
||||
return_stmt ::= ret_expr RETURN_VALUE POP_TOP
|
||||
return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
|
||||
|
||||
iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK come_from_pop
|
||||
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_from_pop
|
||||
|
||||
lastc_stmt ::= iflaststmt COME_FROM
|
||||
|
||||
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
|
||||
|
||||
ifstmt ::= testexpr_then _ifstmts_jump
|
||||
|
||||
# Semantic actions want the else to be at position 3
|
||||
ifelsestmt ::= testexpr c_stmts_opt jf_cf_pop else_suite come_froms
|
||||
ifelsestmt ::= testexpr_then c_stmts_opt jf_cf_pop else_suite come_froms
|
||||
ifelsestmt ::= testexpr c_stmts_opt filler else_suitel come_froms POP_TOP
|
||||
ifelsestmt ::= testexpr_then c_stmts_opt filler else_suitel come_froms POP_TOP
|
||||
|
||||
# Semantic actions want else_suitel to be at index 3
|
||||
ifelsestmtl ::= testexpr_then c_stmts_opt jb_cf_pop else_suitel
|
||||
ifelsestmtc ::= testexpr_then c_stmts_opt ja_cf_pop else_suitec
|
||||
|
||||
iflaststmt ::= testexpr_then c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
|
||||
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE come_froms POP_TOP
|
||||
|
||||
testexpr_then ::= testtrue_then
|
||||
testexpr_then ::= testfalse_then
|
||||
testtrue_then ::= expr jmp_true_then
|
||||
testfalse_then ::= expr jmp_false_then
|
||||
|
||||
jmp_false_then ::= JUMP_IF_FALSE THEN POP_TOP
|
||||
jmp_true_then ::= JUMP_IF_TRUE THEN POP_TOP
|
||||
|
||||
# Common with 2.7
|
||||
while1stmt ::= SETUP_LOOP return_stmts bp_come_from
|
||||
while1stmt ::= SETUP_LOOP return_stmts COME_FROM
|
||||
@@ -172,32 +193,35 @@ class Python26Parser(Python2Parser):
|
||||
|
||||
list_iter ::= list_if JUMP_BACK
|
||||
list_iter ::= list_if JUMP_BACK COME_FROM POP_TOP
|
||||
list_compr ::= BUILD_LIST_0 DUP_TOP
|
||||
designator list_iter del_stmt
|
||||
list_compr ::= BUILD_LIST_0 DUP_TOP
|
||||
designator list_iter JUMP_BACK del_stmt
|
||||
lc_body ::= LOAD_NAME expr LIST_APPEND
|
||||
list_compr ::= BUILD_LIST_0 DUP_TOP
|
||||
designator list_iter del_stmt
|
||||
list_compr ::= BUILD_LIST_0 DUP_TOP
|
||||
designator list_iter JUMP_BACK del_stmt
|
||||
lc_body ::= LOAD_NAME expr LIST_APPEND
|
||||
lc_body ::= LOAD_FAST expr LIST_APPEND
|
||||
|
||||
comp_for ::= SETUP_LOOP expr _for designator comp_iter jb_bp_come_from
|
||||
|
||||
comp_body ::= gen_comp_body
|
||||
|
||||
for_block ::= l_stmts_opt _come_from POP_TOP JUMP_BACK
|
||||
|
||||
# Make sure we keep indices the same as 2.7
|
||||
setup_loop_lf ::= SETUP_LOOP LOAD_FAST
|
||||
genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter jb_bp_come_from
|
||||
genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop jb_bp_come_from
|
||||
genexpr_func ::= setup_loop_lf FOR_ITER designator comp_iter JUMP_BACK come_from_pop
|
||||
jb_bp_come_from
|
||||
genexpr ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
|
||||
|
||||
list_if ::= list_if ::= expr jmp_false_then list_iter
|
||||
'''
|
||||
|
||||
def p_ret26(self, args):
|
||||
'''
|
||||
ret_and ::= expr jmp_false ret_expr_or_cond COME_FROM
|
||||
ret_or ::= expr jmp_true ret_expr_or_cond COME_FROM
|
||||
ret_cond ::= expr jmp_false expr RETURN_END_IF POP_TOP ret_expr_or_cond
|
||||
ret_cond ::= expr jmp_false expr ret_expr_or_cond
|
||||
ret_cond_not ::= expr jmp_true expr RETURN_END_IF POP_TOP ret_expr_or_cond
|
||||
ret_and ::= expr jmp_false ret_expr_or_cond COME_FROM
|
||||
ret_or ::= expr jmp_true ret_expr_or_cond COME_FROM
|
||||
ret_cond ::= expr jmp_false_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
|
||||
ret_cond ::= expr jmp_false_then expr ret_expr_or_cond
|
||||
ret_cond_not ::= expr jmp_true_then expr RETURN_END_IF POP_TOP ret_expr_or_cond
|
||||
|
||||
return_if_stmt ::= ret_expr RETURN_END_IF POP_TOP
|
||||
return_stmt ::= ret_expr RETURN_VALUE POP_TOP
|
||||
@@ -207,17 +231,37 @@ class Python26Parser(Python2Parser):
|
||||
'''
|
||||
|
||||
def p_except26(self, args):
|
||||
'''
|
||||
except_suite ::= c_stmts_opt jmp_abs come_from_pop
|
||||
'''
|
||||
"""
|
||||
except_suite ::= c_stmts_opt jmp_abs POP_TOP
|
||||
"""
|
||||
|
||||
def p_misc26(self, args):
|
||||
'''
|
||||
"""
|
||||
conditional ::= expr jmp_false expr jf_cf_pop expr come_from_opt
|
||||
and ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
|
||||
cmp_list ::= expr cmp_list1 ROT_TWO COME_FROM POP_TOP _come_from
|
||||
'''
|
||||
|
||||
conditional_lambda ::= expr jmp_false_then return_if_stmt return_stmt LAMBDA_MARKER
|
||||
"""
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
super(Python26Parser, self).add_custom_rules(tokens, customize)
|
||||
self.check_reduce['and'] = 'AST'
|
||||
|
||||
def reduce_is_invalid(self, rule, ast, tokens, first, last):
|
||||
invalid = super(Python26Parser,
|
||||
self).reduce_is_invalid(rule, ast,
|
||||
tokens, first, last)
|
||||
if invalid:
|
||||
return invalid
|
||||
if rule == ('and', ('expr', 'jmp_false', 'expr', '\\e_come_from_opt')):
|
||||
# Test that jmp_false jumps to the end of "and"
|
||||
# or that it jumps to the same place as the end of "and"
|
||||
jmp_false = ast[1][0]
|
||||
jmp_target = jmp_false.offset + jmp_false.attr + 3
|
||||
return not (jmp_target == tokens[last].offset or
|
||||
tokens[last].pattr == jmp_false.pattr)
|
||||
return False
|
||||
class Python26ParserSingle(Python2Parser, PythonParserSingle):
|
||||
pass
|
||||
|
||||
@@ -237,8 +281,8 @@ if __name__ == '__main__':
|
||||
""".split()))
|
||||
remain_tokens = set(tokens) - opcode_set
|
||||
import re
|
||||
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
|
||||
remain_tokens = set(remain_tokens) - opcode_set
|
||||
print(remain_tokens)
|
||||
# print(sorted(p.rule2name.items()))
|
||||
|
@@ -31,6 +31,10 @@ class Python27Parser(Python2Parser):
|
||||
|
||||
def p_try27(self, args):
|
||||
"""
|
||||
tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST
|
||||
COME_FROM_FINALLY suite_stmts_opt END_FINALLY
|
||||
|
||||
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
|
||||
try_middle else_suite COME_FROM
|
||||
|
||||
@@ -45,7 +49,10 @@ class Python27Parser(Python2Parser):
|
||||
|
||||
def p_jump27(self, args):
|
||||
"""
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
|
||||
come_froms ::= come_froms COME_FROM
|
||||
come_froms ::= COME_FROM
|
||||
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD come_froms
|
||||
bp_come_from ::= POP_BLOCK COME_FROM
|
||||
|
||||
# FIXME: Common with 3.0+
|
||||
@@ -54,7 +61,7 @@ class Python27Parser(Python2Parser):
|
||||
|
||||
ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
|
||||
ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
|
||||
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
|
||||
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM ret_expr_or_cond
|
||||
ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
|
||||
|
||||
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
|
||||
@@ -77,15 +84,13 @@ class Python27Parser(Python2Parser):
|
||||
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2
|
||||
|
||||
withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP END_FINALLY
|
||||
|
||||
withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP END_FINALLY
|
||||
|
||||
while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
|
||||
|
||||
# Common with 2.6
|
||||
while1stmt ::= SETUP_LOOP return_stmts bp_come_from
|
||||
while1stmt ::= SETUP_LOOP return_stmts COME_FROM
|
||||
@@ -110,8 +115,10 @@ if __name__ == '__main__':
|
||||
""".split()))
|
||||
remain_tokens = set(tokens) - opcode_set
|
||||
import re
|
||||
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_\d+$', '', t)
|
||||
for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$', '', t)
|
||||
for t in remain_tokens])
|
||||
remain_tokens = set(remain_tokens) - opcode_set
|
||||
print(remain_tokens)
|
||||
# p.dumpGrammar()
|
||||
|
@@ -15,8 +15,6 @@ If we succeed in creating a parse tree, then we have a Python program
|
||||
that a later phase can turn into a sequence of ASCII text.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
|
||||
from uncompyle6.parsers.astnode import AST
|
||||
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
|
||||
@@ -44,6 +42,10 @@ class Python3Parser(PythonParser):
|
||||
|
||||
list_for ::= expr FOR_ITER designator list_iter jb_or_c
|
||||
|
||||
# This is seen in PyPy, but possibly it appears on other Python 3?
|
||||
list_if ::= expr jmp_false list_iter COME_FROM
|
||||
list_if_not ::= expr jmp_true list_iter COME_FROM
|
||||
|
||||
jb_or_c ::= JUMP_BACK
|
||||
jb_or_c ::= CONTINUE
|
||||
|
||||
@@ -52,6 +54,9 @@ class Python3Parser(PythonParser):
|
||||
setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
|
||||
JUMP_BACK RETURN_VALUE RETURN_LAST
|
||||
|
||||
setcomp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER designator comp_iter
|
||||
COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST
|
||||
|
||||
comp_body ::= dict_comp_body
|
||||
comp_body ::= set_comp_body
|
||||
dict_comp_body ::= expr expr MAP_ADD
|
||||
@@ -100,8 +105,7 @@ class Python3Parser(PythonParser):
|
||||
del_stmt ::= expr DELETE_ATTR
|
||||
|
||||
kwarg ::= LOAD_CONST expr
|
||||
kwargs ::= kwargs kwarg
|
||||
kwargs ::=
|
||||
kwargs ::= kwarg*
|
||||
|
||||
classdef ::= build_class designator
|
||||
|
||||
@@ -114,9 +118,11 @@ class Python3Parser(PythonParser):
|
||||
classdefdeco1 ::= expr classdefdeco1 CALL_FUNCTION_1
|
||||
classdefdeco1 ::= expr classdefdeco2 CALL_FUNCTION_1
|
||||
|
||||
assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
|
||||
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
|
||||
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr RAISE_VARARGS_2
|
||||
assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM
|
||||
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1
|
||||
RAISE_VARARGS_1 COME_FROM
|
||||
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr
|
||||
RAISE_VARARGS_2 COME_FROM
|
||||
|
||||
assert_expr ::= expr
|
||||
assert_expr ::= assert_expr_or
|
||||
@@ -133,22 +139,31 @@ class Python3Parser(PythonParser):
|
||||
|
||||
_ifstmts_jump ::= return_if_stmts
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM
|
||||
_ifstmts_jump ::= c_stmts_opt COME_FROM
|
||||
|
||||
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE
|
||||
|
||||
iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK
|
||||
iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
|
||||
|
||||
# These are used to keep AST indices the same
|
||||
jf_else ::= JUMP_FORWARD ELSE
|
||||
ja_else ::= JUMP_ABSOLUTE ELSE
|
||||
|
||||
# Note: in if/else kinds of statements, we err on the side
|
||||
# of missing "else" clauses. Therefore we include grammar
|
||||
# rules with and without ELSE.
|
||||
|
||||
ifelsestmt ::= testexpr c_stmts_opt JUMP_FORWARD else_suite COME_FROM
|
||||
ifelsestmt ::= testexpr c_stmts_opt jf_else else_suite _come_from
|
||||
|
||||
ifelsestmtc ::= testexpr c_stmts_opt JUMP_ABSOLUTE else_suitec
|
||||
ifelsestmtc ::= testexpr c_stmts_opt ja_else else_suitec
|
||||
|
||||
ifelsestmtr ::= testexpr return_if_stmts return_stmts
|
||||
|
||||
ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
|
||||
ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel JUMP_BACK COME_FROM_LOOP
|
||||
ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel COME_FROM_LOOP
|
||||
|
||||
ifelsestmtl ::= testexpr c_stmts_opt COME_FROM JUMP_BACK else_suitel
|
||||
|
||||
# FIXME: this feels like a hack. Is it just 1 or two
|
||||
# COME_FROMs? the parsed tree for this and even with just the
|
||||
@@ -246,7 +261,6 @@ class Python3Parser(PythonParser):
|
||||
c_stmts_opt34 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
|
||||
"""
|
||||
|
||||
|
||||
def p_def_annotations3(self, args):
|
||||
"""
|
||||
# Annotated functions
|
||||
@@ -324,6 +338,9 @@ class Python3Parser(PythonParser):
|
||||
forelselaststmtl ::= SETUP_LOOP expr _for designator for_block POP_BLOCK else_suitel
|
||||
COME_FROM_LOOP
|
||||
|
||||
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
|
||||
COME_FROM_LOOP
|
||||
|
||||
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
|
||||
COME_FROM_LOOP
|
||||
|
||||
@@ -335,11 +352,12 @@ class Python3Parser(PythonParser):
|
||||
whilestmt ::= SETUP_LOOP testexpr return_stmts POP_BLOCK
|
||||
COME_FROM_LOOP
|
||||
|
||||
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK
|
||||
else_suite
|
||||
|
||||
whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
|
||||
else_suite COME_FROM_LOOP
|
||||
|
||||
while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK
|
||||
else_suite
|
||||
|
||||
whileelselaststmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
|
||||
else_suitec COME_FROM_LOOP
|
||||
@@ -348,14 +366,16 @@ class Python3Parser(PythonParser):
|
||||
|
||||
# FIXME: Python 3.? starts adding branch optimization? Put this starting there.
|
||||
while1stmt ::= SETUP_LOOP l_stmts
|
||||
while1stmt ::= SETUP_LOOP l_stmts COME_FROM_LOOP
|
||||
|
||||
while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
|
||||
|
||||
|
||||
# FIXME: investigate - can code really produce a NOP?
|
||||
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
|
||||
COME_FROM_LOOP
|
||||
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP
|
||||
COME_FROM_LOOP
|
||||
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP
|
||||
COME_FROM_LOOP
|
||||
forstmt ::= SETUP_LOOP expr _for designator for_block POP_BLOCK NOP
|
||||
COME_FROM_LOOP
|
||||
"""
|
||||
@@ -370,16 +390,17 @@ class Python3Parser(PythonParser):
|
||||
'''
|
||||
|
||||
def p_expr3(self, args):
|
||||
'''
|
||||
"""
|
||||
conditional ::= expr jmp_false expr jf_else expr COME_FROM
|
||||
conditionalnot ::= expr jmp_true expr jf_else expr COME_FROM
|
||||
|
||||
|
||||
expr ::= LOAD_CLASSNAME
|
||||
|
||||
# Python 3.4+
|
||||
expr ::= LOAD_CLASSDEREF
|
||||
|
||||
binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
|
||||
# Python3 drops slice0..slice3
|
||||
|
||||
'''
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def call_fn_name(token):
|
||||
@@ -442,11 +463,20 @@ class Python3Parser(PythonParser):
|
||||
args_kw = (token.attr >> 8) & 0xff
|
||||
nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
|
||||
token.type = self.call_fn_name(token)
|
||||
rule = ('call_function ::= expr '
|
||||
+ ('pos_arg ' * args_pos)
|
||||
+ ('kwarg ' * args_kw)
|
||||
+ 'expr ' * nak + token.type)
|
||||
rule = ('call_function ::= expr ' +
|
||||
('pos_arg ' * args_pos) +
|
||||
('kwarg ' * args_kw) +
|
||||
'expr ' * nak + token.type)
|
||||
self.add_unique_rule(rule, token.type, args_pos, customize)
|
||||
if self.version >= 3.5:
|
||||
rule = ('async_call_function ::= expr ' +
|
||||
('pos_arg ' * args_pos) +
|
||||
('kwarg ' * args_kw) +
|
||||
'expr ' * nak + token.type +
|
||||
' GET_AWAITABLE LOAD_CONST YIELD_FROM')
|
||||
self.add_unique_rule(rule, token.type, args_pos, customize)
|
||||
self.add_unique_rule('expr ::= async_call_function', token.type, args_pos, customize)
|
||||
|
||||
rule = ('classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d'
|
||||
% (('expr ' * (args_pos-1)), opname, args_pos))
|
||||
self.add_unique_rule(rule, token.type, args_pos, customize)
|
||||
@@ -455,7 +485,10 @@ class Python3Parser(PythonParser):
|
||||
"""Python 3.3 added a an addtional LOAD_CONST before MAKE_FUNCTION and
|
||||
this has an effect on many rules.
|
||||
"""
|
||||
new_rule = rule % (('LOAD_CONST ') * (1 if self.version >= 3.3 else 0))
|
||||
if self.version >= 3.3:
|
||||
new_rule = rule % (('LOAD_CONST ') * 1)
|
||||
else:
|
||||
new_rule = rule % (('LOAD_CONST ') * 0)
|
||||
self.add_unique_rule(new_rule, opname, attr, customize)
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
@@ -470,8 +503,15 @@ class Python3Parser(PythonParser):
|
||||
|
||||
# build_class (see load_build_class)
|
||||
|
||||
build_list ::= {expr}^n BUILD_LIST_n
|
||||
build_list ::= {expr}^n BUILD_TUPLE_n
|
||||
# Even the below say _list, in the semantic rules we
|
||||
# disambiguate tuples, and sets from lists
|
||||
|
||||
build_list ::= {expr}^n BUILD_LIST_n
|
||||
build_list ::= {expr}^n BUILD_TUPLE_n
|
||||
build_list ::= {expr}^n BUILD_SET_n
|
||||
build_list ::= {expr}^n BUILD_LIST_UNPACK_n
|
||||
build_list ::= {expr}^n BUILD_SET_UNPACK_n
|
||||
build_list ::= {expr}^n BUILD_TUPLE_UNPACK_n
|
||||
|
||||
load_closure ::= {LOAD_CLOSURE}^n BUILD_TUPLE_n
|
||||
# call_function (see custom_classfunc_rule)
|
||||
@@ -544,7 +584,8 @@ class Python3Parser(PythonParser):
|
||||
elif opname_base in ('BUILD_LIST', 'BUILD_TUPLE', 'BUILD_SET'):
|
||||
v = token.attr
|
||||
rule = ('build_list ::= ' + 'expr1024 ' * int(v//1024) +
|
||||
'expr32 ' * int((v//32)%32) + 'expr '*(v%32) + opname)
|
||||
'expr32 ' * int((v//32) % 32) +
|
||||
'expr ' * (v % 32) + opname)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
if opname_base == 'BUILD_TUPLE':
|
||||
rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname))
|
||||
@@ -584,14 +625,21 @@ class Python3Parser(PythonParser):
|
||||
self.add_unique_rule(rule, 'kvlist_n', 1, customize)
|
||||
rule = "mapexpr ::= BUILD_MAP_n kvlist_n"
|
||||
elif self.version >= 3.5:
|
||||
rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
rule = "mapexpr ::= %s %s" % (kvlist_n, opname)
|
||||
if opname != 'BUILD_MAP_WITH_CALL':
|
||||
rule = kvlist_n + ' ::= ' + 'expr ' * (token.attr*2)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
rule = "mapexpr ::= %s %s" % (kvlist_n, opname)
|
||||
|
||||
else:
|
||||
rule = kvlist_n + ' ::= ' + 'expr expr STORE_MAP ' * token.attr
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
rule = "mapexpr ::= %s %s" % (opname, kvlist_n)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
elif opname_base == 'BUILD_CONST_KEY_MAP':
|
||||
# This is in 3.6+
|
||||
kvlist_n = 'expr ' * (token.attr)
|
||||
rule = "mapexpr ::= %sLOAD_CONST %s" % (kvlist_n, opname)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
elif opname_base in ('UNPACK_EX',):
|
||||
before_count, after_count = token.attr
|
||||
rule = 'unpack ::= ' + opname + ' designator' * (before_count + after_count + 1)
|
||||
@@ -628,6 +676,27 @@ class Python3Parser(PythonParser):
|
||||
rule = ('mkfunc ::= kwargs %sexpr %s' %
|
||||
('pos_arg ' * args_pos, opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
if opname.startswith('MAKE_FUNCTION_A'):
|
||||
# rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
|
||||
# ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
if self.version >= 3.3:
|
||||
rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
|
||||
(('pos_arg ' * (args_pos)),
|
||||
('call_function ' * (annotate_args-1)), opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST LOAD_CONST EXTENDED_ARG %s' %
|
||||
(('pos_arg ' * (args_pos)),
|
||||
('annotate_arg ' * (annotate_args-1)), opname))
|
||||
else:
|
||||
rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
|
||||
(('pos_arg ' * (args_pos)),
|
||||
('annotate_arg ' * (annotate_args-1)), opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
|
||||
(('pos_arg ' * (args_pos)),
|
||||
('call_function ' * (annotate_args-1)), opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
elif opname_base == 'CALL_METHOD':
|
||||
# PyPy only - DRY with parse2
|
||||
|
||||
@@ -637,10 +706,10 @@ class Python3Parser(PythonParser):
|
||||
|
||||
# number of apply equiv arguments:
|
||||
nak = ( len(opname_base)-len('CALL_METHOD') ) // 3
|
||||
rule = ('call_function ::= expr '
|
||||
+ ('pos_arg ' * args_pos)
|
||||
+ ('kwarg ' * args_kw)
|
||||
+ 'expr ' * nak + opname)
|
||||
rule = ('call_function ::= expr ' +
|
||||
('pos_arg ' * args_pos) +
|
||||
('kwarg ' * args_kw) +
|
||||
'expr ' * nak + opname)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
elif opname.startswith('MAKE_CLOSURE'):
|
||||
# DRY with MAKE_FUNCTION
|
||||
@@ -684,8 +753,46 @@ class Python3Parser(PythonParser):
|
||||
rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'
|
||||
% ('expr ' * args_pos, opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
pass
|
||||
self.check_reduce['augassign1'] = 'AST'
|
||||
self.check_reduce['augassign2'] = 'AST'
|
||||
self.check_reduce['while1stmt'] = 'noAST'
|
||||
self.check_reduce['annotate_tuple'] = 'noAST'
|
||||
self.check_reduce['kwarg'] = 'noAST'
|
||||
# FIXME: remove parser errors caused by the below
|
||||
# self.check_reduce['while1elsestmt'] = 'noAST'
|
||||
return
|
||||
|
||||
def reduce_is_invalid(self, rule, ast, tokens, first, last):
|
||||
lhs = rule[0]
|
||||
if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
|
||||
return True
|
||||
elif lhs == 'annotate_tuple':
|
||||
return not isinstance(tokens[first].attr, tuple)
|
||||
elif lhs == 'kwarg':
|
||||
return not (isinstance(tokens[first].attr, unicode) or
|
||||
isinstance(tokens[first].attr, str))
|
||||
elif lhs == 'while1elsestmt':
|
||||
# if SETUP_LOOP target spans the else part, then this is
|
||||
# not while1else. Also do for whileTrue?
|
||||
last += 1
|
||||
while isinstance(tokens[last].offset, str):
|
||||
last += 1
|
||||
return tokens[first].attr == tokens[last].offset
|
||||
elif lhs == 'while1stmt':
|
||||
if tokens[last] in ('COME_FROM_LOOP', 'JUMP_BACK'):
|
||||
# jump_back should be right afer SETUP_LOOP. Test?
|
||||
last += 1
|
||||
while last < len(tokens) and isinstance(tokens[last].offset, str):
|
||||
last += 1
|
||||
if last < len(tokens):
|
||||
offset = tokens[last].offset
|
||||
assert tokens[first] == 'SETUP_LOOP'
|
||||
if offset != tokens[first].attr:
|
||||
return True
|
||||
return False
|
||||
return False
|
||||
|
||||
class Python30Parser(Python3Parser):
|
||||
|
||||
def p_30(self, args):
|
||||
|
@@ -2,12 +2,11 @@
|
||||
"""
|
||||
spark grammar differences over Python 3.1 for Python 3.0.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParserSingle
|
||||
from uncompyle6.parsers.parse3 import Python3Parser
|
||||
from uncompyle6.parsers.parse31 import Python31Parser
|
||||
|
||||
class Python30Parser(Python3Parser):
|
||||
class Python30Parser(Python31Parser):
|
||||
|
||||
def p_30(self, args):
|
||||
"""
|
||||
@@ -15,6 +14,11 @@ class Python30Parser(Python3Parser):
|
||||
stmt ::= store_locals
|
||||
store_locals ::= LOAD_FAST STORE_LOCALS
|
||||
|
||||
# FIXME: combine with parse3.2
|
||||
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK
|
||||
COME_FROM_LOOP
|
||||
whileTruestmt ::= SETUP_LOOP return_stmts
|
||||
COME_FROM_LOOP
|
||||
|
||||
# In many ways Python 3.0 code generation is more like Python 2.6 than
|
||||
# it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below
|
||||
@@ -39,5 +43,10 @@ class Python30Parser(Python3Parser):
|
||||
setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
|
||||
"""
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
super(Python30Parser, self).add_custom_rules(tokens, customize)
|
||||
return
|
||||
pass
|
||||
|
||||
class Python30ParserSingle(Python30Parser, PythonParserSingle):
|
||||
pass
|
||||
|
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
spark grammar differences over Python 3.2 for Python 3.1.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParserSingle
|
||||
from uncompyle6.parsers.parse32 import Python32Parser
|
||||
@@ -36,16 +35,8 @@ class Python31Parser(Python32Parser):
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
super(Python31Parser, self).add_custom_rules(tokens, customize)
|
||||
for i, token in enumerate(tokens):
|
||||
opname = token.type
|
||||
if opname.startswith('MAKE_FUNCTION_A'):
|
||||
args_pos, args_kw, annotate_args = token.attr
|
||||
# Check that there are 2 annotated params?
|
||||
# rule = ('mkfunc2 ::= %s%sEXTENDED_ARG %s' %
|
||||
# ('pos_arg ' * (args_pos), 'kwargs ' * (annotate_args-1), opname))
|
||||
rule = ('mkfunc_annotate ::= %s%sannotate_tuple LOAD_CONST EXTENDED_ARG %s' %
|
||||
(('pos_arg ' * (args_pos)),
|
||||
('annotate_arg ' * (annotate_args-1)), opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
return
|
||||
pass
|
||||
|
||||
class Python31ParserSingle(Python31Parser, PythonParserSingle):
|
||||
pass
|
||||
|
@@ -2,17 +2,12 @@
|
||||
"""
|
||||
spark grammar differences over Python 3 for Python 3.2.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParserSingle
|
||||
from uncompyle6.parsers.parse3 import Python3Parser
|
||||
|
||||
class Python32Parser(Python3Parser):
|
||||
def p_32to35(self, args):
|
||||
"""
|
||||
# In Python 3.2+, DUP_TOPX is DUP_TOP_TWO
|
||||
binary_subscr2 ::= expr expr DUP_TOP_TWO BINARY_SUBSCR
|
||||
|
||||
# Store locals is only in Python 3.0 to 3.3
|
||||
stmt ::= store_locals
|
||||
store_locals ::= LOAD_FAST STORE_LOCALS
|
||||
@@ -22,6 +17,11 @@ class Python32Parser(Python3Parser):
|
||||
COME_FROM_LOOP
|
||||
whileTruestmt ::= SETUP_LOOP return_stmts
|
||||
COME_FROM_LOOP
|
||||
|
||||
# Python 3.2+ has more loop optimization that removes
|
||||
# JUMP_FORWARD in some cases, and hence we also don't
|
||||
# see COME_FROM
|
||||
_ifstmts_jump ::= c_stmts_opt
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -44,6 +44,9 @@ class Python32Parser(Python3Parser):
|
||||
(('pos_arg ' * (args_pos)),
|
||||
('annotate_arg ' * (annotate_args-1)), opname))
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
pass
|
||||
return
|
||||
pass
|
||||
|
||||
|
||||
class Python32ParserSingle(Python32Parser, PythonParserSingle):
|
||||
|
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
spark grammar differences over Python 3.2 for Python 3.3.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParserSingle
|
||||
from uncompyle6.parsers.parse32 import Python32Parser
|
||||
@@ -19,14 +18,8 @@ class Python33Parser(Python32Parser):
|
||||
# actions that want c_stmts_opt at index 1
|
||||
|
||||
iflaststmt ::= testexpr c_stmts_opt33
|
||||
iflaststmtl ::= testexpr c_stmts_opt
|
||||
c_stmts_opt33 ::= JUMP_BACK JUMP_ABSOLUTE c_stmts_opt
|
||||
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_from
|
||||
|
||||
# Python 3.3+ has more loop optimization that removes
|
||||
# JUMP_FORWARD in some cases, and hence we also don't
|
||||
# see COME_FROM
|
||||
_ifstmts_jump ::= c_stmts_opt
|
||||
"""
|
||||
|
||||
class Python33ParserSingle(Python33Parser, PythonParserSingle):
|
||||
|
@@ -17,8 +17,6 @@ class Python34Parser(Python33Parser):
|
||||
"""
|
||||
# Python 3.4+ optimizes the trailing two JUMPS away
|
||||
|
||||
for_block ::= l_stmts
|
||||
|
||||
# Is this 3.4 only?
|
||||
yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
|
||||
"""
|
||||
@@ -42,8 +40,8 @@ if __name__ == '__main__':
|
||||
""".split()))
|
||||
remain_tokens = set(tokens) - opcode_set
|
||||
import re
|
||||
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
|
||||
remain_tokens = set(remain_tokens) - opcode_set
|
||||
print(remain_tokens)
|
||||
# print(sorted(p.rule2name.items()))
|
||||
|
@@ -2,7 +2,6 @@
|
||||
"""
|
||||
spark grammar differences over Python 3.4 for Python 3.5.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
|
||||
from uncompyle6.parser import PythonParserSingle
|
||||
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
|
||||
@@ -16,43 +15,124 @@ class Python35Parser(Python34Parser):
|
||||
|
||||
def p_35on(self, args):
|
||||
"""
|
||||
# The number of canned instructions in new statements is mind boggling.
|
||||
# I'm sure by the time Python 4 comes around these will be turned
|
||||
# into special opcodes
|
||||
|
||||
while1stmt ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK
|
||||
POP_BLOCK COME_FROM_LOOP
|
||||
|
||||
# Python 3.5+ Await statement
|
||||
stmt ::= await_stmt
|
||||
await_stmt ::= call_function GET_AWAITABLE LOAD_CONST YIELD_FROM POP_TOP
|
||||
|
||||
# Python 3.5+ has WITH_CLEANUP_START/FINISH
|
||||
|
||||
withstmt ::= expr SETUP_WITH exprlist suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
|
||||
withstmt ::= expr
|
||||
SETUP_WITH exprlist suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
|
||||
|
||||
withstmt ::= expr SETUP_WITH POP_TOP suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
|
||||
withstmt ::= expr
|
||||
SETUP_WITH POP_TOP suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
|
||||
|
||||
withasstmt ::= expr
|
||||
SETUP_WITH designator suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
|
||||
|
||||
|
||||
# Python 3.5+ async additions
|
||||
|
||||
stmt ::= async_with_stmt
|
||||
async_with_stmt ::= expr
|
||||
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
|
||||
SETUP_ASYNC_WITH POP_TOP suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST
|
||||
WITH_CLEANUP_START
|
||||
GET_AWAITABLE LOAD_CONST YIELD_FROM
|
||||
WITH_CLEANUP_FINISH END_FINALLY
|
||||
|
||||
stmt ::= async_with_as_stmt
|
||||
async_with_as_stmt ::= expr
|
||||
BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM
|
||||
SETUP_ASYNC_WITH designator suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST
|
||||
WITH_CLEANUP_START
|
||||
GET_AWAITABLE LOAD_CONST YIELD_FROM
|
||||
WITH_CLEANUP_FINISH END_FINALLY
|
||||
|
||||
|
||||
stmt ::= async_for_stmt
|
||||
async_for_stmt ::= SETUP_LOOP expr
|
||||
GET_AITER
|
||||
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
|
||||
YIELD_FROM
|
||||
designator
|
||||
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
|
||||
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
|
||||
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
|
||||
JUMP_ABSOLUTE END_FINALLY COME_FROM
|
||||
for_block POP_BLOCK JUMP_ABSOLUTE
|
||||
opt_come_from_loop
|
||||
|
||||
stmt ::= async_forelse_stmt
|
||||
async_forelse_stmt ::= SETUP_LOOP expr
|
||||
GET_AITER
|
||||
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
|
||||
YIELD_FROM
|
||||
designator
|
||||
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
|
||||
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
|
||||
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
|
||||
JUMP_ABSOLUTE END_FINALLY COME_FROM
|
||||
for_block POP_BLOCK JUMP_ABSOLUTE
|
||||
else_suite COME_FROM_LOOP
|
||||
|
||||
withasstmt ::= expr SETUP_WITH designator suite_stmts_opt
|
||||
POP_BLOCK LOAD_CONST COME_FROM_WITH
|
||||
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
|
||||
|
||||
inplace_op ::= INPLACE_MATRIX_MULTIPLY
|
||||
binary_op ::= BINARY_MATRIX_MULTIPLY
|
||||
|
||||
# Python 3.5+ does jump optimization
|
||||
# In <.3.5 the below is a JUMP_FORWARD to a JUMP_ABSOLUTE.
|
||||
# in return_stmt, we will need the semantic actions in pysource.py
|
||||
# to work out whether to dedent or not based on the presence of
|
||||
# RETURN_END_IF vs RETURN_VALUE
|
||||
|
||||
return_if_stmt ::= ret_expr RETURN_END_IF POP_BLOCK
|
||||
|
||||
ifelsestmtc ::= testexpr c_stmts_opt JUMP_FORWARD else_suitec
|
||||
ifelsestmtc ::= testexpr c_stmts_opt jf_else else_suitec
|
||||
|
||||
# ifstmt ::= testexpr c_stmts_opt
|
||||
|
||||
iflaststmt ::= testexpr c_stmts_opt JUMP_FORWARD
|
||||
|
||||
# Python 3.3+ also has yield from. 3.5 does it
|
||||
# differently than 3.3, 3.4
|
||||
|
||||
expr ::= yield_from
|
||||
yield_from ::= expr GET_YIELD_FROM_ITER LOAD_CONST YIELD_FROM
|
||||
|
||||
# Python 3.4+ has more loop optimization that removes
|
||||
# JUMP_FORWARD in some cases, and hence we also don't
|
||||
# see COME_FROM
|
||||
_ifstmts_jump ::= c_stmts_opt
|
||||
"""
|
||||
|
||||
def add_custom_rules(self, tokens, customize):
|
||||
super(Python35Parser, self).add_custom_rules(tokens, customize)
|
||||
for i, token in enumerate(tokens):
|
||||
opname = token.type
|
||||
if opname == 'BUILD_MAP_UNPACK_WITH_CALL':
|
||||
nargs = token.attr % 256
|
||||
map_unpack_n = "map_unpack_%s" % nargs
|
||||
rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
rule = "unmapexpr ::= %s %s" % (map_unpack_n, opname)
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
call_token = tokens[i+1]
|
||||
if self.version == 3.5:
|
||||
rule = 'call_function ::= expr unmapexpr ' + call_token.type
|
||||
self.add_unique_rule(rule, opname, token.attr, customize)
|
||||
pass
|
||||
pass
|
||||
return
|
||||
|
||||
class Python35ParserSingle(Python35Parser, PythonParserSingle):
|
||||
pass
|
||||
|
||||
@@ -72,8 +152,8 @@ if __name__ == '__main__':
|
||||
""".split()))
|
||||
remain_tokens = set(tokens) - opcode_set
|
||||
import re
|
||||
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
|
||||
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
|
||||
remain_tokens = set(remain_tokens) - opcode_set
|
||||
print(remain_tokens)
|
||||
# print(sorted(p.rule2name.items()))
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user