jinx/python-uncompyle6
1
Fork 0
mirror of https://github.com/rocky/python-uncompyle6.git synced 2025-08-02 16:44:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-2.4

Browse Source
This commit is contained in:
rocky
2019-11-16 18:01:23 -05:00
parent efd8fe54b2 df5df9364c
commit 2c33a06535
26 changed files with 417 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@@ -48,10 +48,14 @@ check-2.4 check-2.5:
# Skip for now
2.6 5.0 5.3 5.6 5.8:
#:PyPy pypy3-2.4.0 Python 3:
#:PyPy pypy3-2.4.0 Python 3.6.1:
7.1 pypy-3.2 2.4:
$(MAKE) -C test $@
#:PyPy pypy3-2.4.0 Python 3.6.9:
7.2:
$(MAKE) -C test $@
#: Run py.test tests
pytest:
$(MAKE) -C pytest check

8
test/Makefile
View File

@@ -337,12 +337,18 @@ pypy-2.7 5.0 5.3 6.0:
pypy-3.2 2.4:
$(PYTHON) test_pythonlib.py --bytecode-pypy3.2 --verify
#: PyPy 5.0.x with Python 3.6 ...
#: PyPy 5.0.x with Python 3.6.1 ...
check-bytecode-pypy3.6: 7.1
7.1:
$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
#: PyPy 5.0.x with Python 3.6.9
check-bytecode-pypy3.6: 7.2
7.2:
$(PYTHON) test_pythonlib.py --bytecode-pypy3.6-run --verify-run
$(PYTHON) test_pythonlib.py --bytecode-pypy3.6 --verify
clean: clean-py-dis clean-dis clean-unverified

BIN
test/bytecode_3.0/02_try_except_except.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.0/02_while1_if_while1.pyc
View File

Binary file not shown.

BIN
test/bytecode_3.0/03_ifelse.pyc Normal file
View File

Binary file not shown.

BIN
test/bytecode_3.0_run/04_and_del.pyc
View File

Binary file not shown.

10
test/simple_source/bug30/02_try_except_except.py
View File

@@ -7,3 +7,13 @@ def start_new_thread(function, args, kwargs={}):
pass
except:
args()
# Adapted from 3.0.1 code.py
# Bug is again JUMP_FORWARD elimination compared
# to earlier and later Pythons.
def interact():
while 1:
try:
more = 1
except KeyboardInterrupt:
more = 0

5
test/simple_source/bug30/02_while1_if_while1.py
View File

@@ -7,3 +7,8 @@ while 1:
raise RuntimeError
else:
raise RuntimeError
# Adapted from 3.0.1 cgi.py
def _parseparam(s, end):
while end > 0 and s.count(''):
end = s.find(';')

50
test/simple_source/bug30/03_ifelse.py Normal file
View File

@@ -0,0 +1,50 @@
# Adapted from 3.0 base64
# Problem was handling if/else which
# needs to be like Python 2.6 (and not like 2.7 or 3.1)
def main(args, f, func, sys):
"""Small main program"""
if args and args[0] != '-':
func(f, sys.stdout.buffer)
else:
func(sys.stdin.buffer, sys.stdout.buffer)
# From Python 3.0 _markupbase.py.
#
# The Problem was in the way "if"s are generated in 3.0 which is sort
# of like a more optimized Python 2.6, with reduced extraneous jumps,
# but still 2.6-ish and not 2.7- or 3.1-ish.
def parse_marked_section(fn, i, rawdata, report=1):
if report:
j = 1
fn(rawdata[i: j])
return 10
# From 3.0.1 _abcoll.py
# Bug was in genexpr_func which doesn't have a JUMP_BACK but
# in its gen_comp_body, we can use COME_FROM in its place.
# As above omission of JUMPs is a feature of 3.0 that doesn't
# seem to be in later versions (or earlier like 2.6).
def __and__(self, other, Iterable):
if not isinstance(other, Iterable):
return NotImplemented
return self._from_iterable(value for value in other if value in self)
# Adapted from 3.0.1 abc.py
# Bug was in handling multiple COME_FROMs in return_if_stmt
def __instancecheck__(subtype, subclass, cls):
if subtype:
if (cls and subclass):
return False
# Adapted from 3.0.1 abc.py
# Bug was rule in "jump_absolute_else" and disallowing
# "else" to the wrong place.
def _strptime(locale_time, found_zone, time):
for tz_values in locale_time:
if found_zone:
if (time and found_zone):
break
else:
break

10
test/simple_source/bug30/04_and_del.py
View File

@@ -18,3 +18,13 @@ assert normpath(['.']) == []
assert normpath(['a', 'b', '..']) == ['a']
assert normpath(['a', 'b', '', 'c']) == ['a', 'b', 'c']
assert normpath(['a', 'b', '.', '', 'c', '..']) == ['a', 'b']
# Adapted from 3.0.1 cgi.py
# Bug was in detecting "or" and "and" due to lack of PUSH/POP_IF instructions.
def handle(format, html, text):
formatter = (format and html) or text
return formatter
assert handle(False, False, True)
assert not handle(True, False, False)
assert handle(True, True, False)

2
test/test_pyenvlib.py
View File

@@ -43,6 +43,8 @@ TEST_VERSIONS = (
"pypy3.5-5.9.0",
"pypy3.5-6.0.0",
"pypy3.6-7.1.0",
"pypy3.6-7.1.1",
"pypy3.6-7.2.0",
"native",
) + tuple(python_versions)

4
uncompyle6/parsers/parse26.py
View File

@@ -254,7 +254,7 @@ class Python26Parser(Python2Parser):
POP_TOP jb_pb_come_from
generator_exp ::= LOAD_GENEXPR MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1 COME_FROM
list_if ::= list_if ::= expr jmp_false_then list_iter
list_if ::= expr jmp_false_then list_iter
'''
def p_ret26(self, args):
@@ -467,7 +467,7 @@ if __name__ == '__main__':
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 2.6:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(

2
uncompyle6/parsers/parse27.py
View File

@@ -292,7 +292,7 @@ if __name__ == '__main__':
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 2.7:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(

17
uncompyle6/parsers/parse3.py
View File

@@ -1439,8 +1439,9 @@ class Python3Parser(PythonParser):
except_handler COME_FROM else_suitel
opt_come_from_except
""",
nop_func,
nop_func
)
custom_ops_processed.add(opname)
elif opname_base in ("UNPACK_EX",):
before_count, after_count = token.attr
@@ -1543,12 +1544,24 @@ class Python3Parser(PythonParser):
for i in range(cfl - 1, first, -1):
if tokens[i] != "POP_BLOCK":
break
if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE"):
if tokens[i].kind not in ("JUMP_BACK", "RETURN_VALUE", "BREAK_LOOP"):
if not tokens[i].kind.startswith("COME_FROM"):
return True
# Check that the SETUP_LOOP jumps to the offset after the
# COME_FROM_LOOP
# Python 3.0 has additional:
# JUMP_FORWARD here
# COME_FROM
# POP_TOP
# COME_FROM
# here:
# (target of SETUP_LOOP)
# We won't check this.
if self.version == 3.0:
return False
if 0 <= last < len(tokens) and tokens[last] in (
"COME_FROM_LOOP",
"JUMP_BACK",

225
uncompyle6/parsers/parse30.py
View File

@@ -11,13 +11,21 @@ class Python30Parser(Python31Parser):
def p_30(self, args):
"""
pt_bp ::= POP_TOP POP_BLOCK
assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP
assert2 ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
come_froms
call_stmt ::= expr _come_froms POP_TOP
return_if_lambda ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP
compare_chained2 ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
# FIXME: combine with parse3.2
whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK
COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt
JUMP_BACK COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP l_stmts_opt
CONTINUE COME_FROM_LOOP
whileTruestmt ::= SETUP_LOOP returns
COME_FROM_LOOP
@@ -42,10 +50,22 @@ class Python30Parser(Python31Parser):
else_suitel ::= l_stmts COME_FROM_LOOP JUMP_BACK
ifelsestmtl ::= testexpr c_stmts_opt jb_pop_top else_suitel
jump_absolute_else ::= COME_FROM JUMP_ABSOLUTE COME_FROM POP_TOP
jump_cf_pop ::= _come_froms _jump _come_froms POP_TOP
ifelsestmt ::= testexpr c_stmts_opt jump_cf_pop else_suite COME_FROM
ifelsestmtl ::= testexpr c_stmts_opt jump_cf_pop else_suitel
ifelsestmtc ::= testexpr c_stmts_opt jump_absolute_else else_suitec
ifelsestmtc ::= testexpr c_stmts_opt jump_cf_pop else_suitec
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM
iflaststmtl ::= testexpr c_stmts_opt jb_pop_top
iflaststmtl ::= testexpr c_stmts_opt come_froms JUMP_BACK COME_FROM POP_TOP
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
withasstmt ::= expr setupwithas store suite_stmts_opt
POP_BLOCK LOAD_CONST COME_FROM_FINALLY
LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
@@ -53,8 +73,8 @@ class Python30Parser(Python31Parser):
setup_finally ::= STORE_FAST SETUP_FINALLY LOAD_FAST DELETE_FAST
# Need to keep LOAD_FAST as index 1
set_comp_func_header ::= BUILD_SET_0 DUP_TOP STORE_FAST
set_comp_func ::= set_comp_func_header
set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST
set_comp_func ::= set_comp_header
LOAD_FAST FOR_ITER store comp_iter
JUMP_BACK POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST
@@ -62,8 +82,10 @@ class Python30Parser(Python31Parser):
list_comp ::= list_comp_header
LOAD_FAST FOR_ITER store comp_iter
JUMP_BACK
list_comp ::= list_comp_header
LOAD_FAST FOR_ITER store comp_iter
JUMP_BACK _come_froms POP_TOP JUMP_BACK
set_comp_header ::= BUILD_SET_0 DUP_TOP STORE_FAST
set_comp ::= set_comp_header
LOAD_FAST FOR_ITER store comp_iter
JUMP_BACK
@@ -72,6 +94,24 @@ class Python30Parser(Python31Parser):
dict_comp ::= dict_comp_header
LOAD_FAST FOR_ITER store dict_comp_iter
JUMP_BACK
dict_comp ::= dict_comp_header
LOAD_FAST FOR_ITER store dict_comp_iter
JUMP_BACK _come_froms POP_TOP JUMP_BACK
stmt ::= try_except30
try_except30 ::= SETUP_EXCEPT suite_stmts_opt
_come_froms pt_bp
except_handler opt_come_from_except
# From Python 2.6
list_iter ::= list_if JUMP_BACK
list_iter ::= list_if JUMP_BACK _come_froms POP_TOP
lc_body ::= LOAD_NAME expr LIST_APPEND
lc_body ::= LOAD_FAST expr LIST_APPEND
list_if ::= expr jmp_false_then list_iter
#############
dict_comp_iter ::= expr expr ROT_TWO expr STORE_SUBSCR
@@ -87,19 +127,52 @@ class Python30Parser(Python31Parser):
except_suite ::= c_stmts POP_EXCEPT jump_except POP_TOP
except_suite_finalize ::= SETUP_FINALLY c_stmts_opt except_var_finalize END_FINALLY
_jump COME_FROM POP_TOP
jump_except ::= JUMP_FORWARD COME_FROM POP_TOP
jump_except ::= JUMP_ABSOLUTE COME_FROM POP_TOP
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts END_FINALLY
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD COME_FROM POP_TOP
_ifstmts_jump ::= c_stmts_opt come_froms POP_TOP JUMP_FORWARD _come_froms
jump_except ::= _jump COME_FROM POP_TOP
or ::= expr jmp_false expr jmp_true expr
or ::= expr jmp_true expr
import_from ::= LOAD_CONST LOAD_CONST IMPORT_NAME importlist _come_froms POP_TOP
################################################################################
# In many ways 3.0 is like 2.6. One similarity is there is no JUMP_IF_TRUE and
# JUMP_IF_FALSE
# The below rules in fact are the same or similar.
jmp_true ::= JUMP_IF_TRUE POP_TOP
jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP
jmp_true ::= JUMP_IF_TRUE POP_TOP
jmp_true_then ::= JUMP_IF_TRUE _come_froms POP_TOP
jmp_false ::= JUMP_IF_FALSE _come_froms POP_TOP
jmp_false_then ::= JUMP_IF_FALSE POP_TOP
# We don't have hacky THEN detection, so we do it
# in the grammar below which is also somewhat hacky.
stmt ::= ifstmt30
stmt ::= ifnotstmt30
ifstmt30 ::= testfalse_then _ifstmts_jump30
ifnotstmt30 ::= testtrue_then _ifstmts_jump30
testfalse_then ::= expr jmp_false_then
testtrue_then ::= expr jmp_true_then
call_stmt ::= expr COME_FROM
_ifstmts_jump30 ::= c_stmts POP_TOP
gen_comp_body ::= expr YIELD_VALUE COME_FROM POP_TOP
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
COME_FROM POP_TOP END_FINALLY
or ::= expr jmp_true_then expr come_from_opt
ret_or ::= expr jmp_true_then expr come_from_opt
ret_and ::= expr jump_false expr come_from_opt
################################################################################
for_block ::= l_stmts_opt _come_froms POP_TOP JUMP_BACK
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
@@ -107,12 +180,17 @@ class Python30Parser(Python31Parser):
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
POP_TOP END_FINALLY
return_if_stmt ::= ret_expr RETURN_END_IF COME_FROM POP_TOP
and ::= expr jmp_false expr come_from_opt
return_if_stmt ::= ret_expr RETURN_END_IF come_froms POP_TOP
return_if_stmt ::= ret_expr RETURN_VALUE come_froms POP_TOP
and ::= expr jmp_false_then expr come_from_opt
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
JUMP_BACK COME_FROM POP_TOP POP_BLOCK COME_FROM_LOOP
JUMP_BACK _come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr returns
POP_TOP POP_BLOCK COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt come_from_opt
come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
# compare_chained is like x <= y <= z
@@ -123,34 +201,149 @@ class Python30Parser(Python31Parser):
compare_chained2 ::= expr COMPARE_OP RETURN_END_IF
"""
def customize_grammar_rules(self, tokens, customize):
super(Python30Parser, self).customize_grammar_rules(tokens, customize)
def remove_rules_30(self):
self.remove_rules("""
iflaststmtl ::= testexpr c_stmts_opt JUMP_BACK COME_FROM_LOOP
ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
iflaststmt ::= testexpr c_stmts_opt JUMP_ABSOLUTE
_ifstmts_jump ::= c_stmts_opt JUMP_FORWARD _come_froms
jump_forward_else ::= JUMP_FORWARD ELSE
jump_absolute_else ::= JUMP_ABSOLUTE ELSE
whilestmt ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
COME_FROM_LOOP
whilestmt ::= SETUP_LOOP testexpr returns
POP_BLOCK COME_FROM_LOOP
assert ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
return_if_lambda ::= RETURN_END_IF_LAMBDA
except_suite ::= c_stmts POP_EXCEPT jump_except
whileelsestmt ::= SETUP_LOOP testexpr l_stmts JUMP_BACK POP_BLOCK
else_suitel COME_FROM_LOOP
# No JUMP_IF_FALSE_OR_POP
# No JUMP_IF_FALSE_OR_POP, JUMP_IF_TRUE_OR_POP,
# POP_JUMP_IF_FALSE, or POP_JUMP_IF_TRUE
jmp_false ::= POP_JUMP_IF_FALSE
jmp_true ::= JUMP_IF_TRUE_OR_POP POP_TOP
jmp_true ::= POP_JUMP_IF_TRUE
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained1 COME_FROM
compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
compare_chained2 COME_FROM
ret_or ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
ret_and ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF
COME_FROM ret_expr_or_cond
ret_expr_or_cond ::= ret_cond
or ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
and ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
""")
def customize_grammar_rules(self, tokens, customize):
super(Python30Parser, self).customize_grammar_rules(tokens, customize)
self.remove_rules_30()
self.check_reduce["iflaststmtl"] = "AST"
self.check_reduce['ifstmt'] = "AST"
self.check_reduce["ifelsestmtc"] = "AST"
self.check_reduce["ifelsestmt"] = "AST"
# self.check_reduce["and"] = "stmt"
return
def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python30Parser,
self).reduce_is_invalid(rule, ast,
tokens, first, last)
if invalid:
return invalid
lhs = rule[0]
if (
lhs in ("iflaststmtl", "ifstmt",
"ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr"
):
testexpr = ast[0]
if testexpr[0] == "testfalse":
testfalse = testexpr[0]
if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else":
jump_absolute_else = ast[2]
come_from = jump_absolute_else[2]
return come_from == "COME_FROM" and come_from.attr < tokens[first].offset
pass
elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop":
jump_cf_pop = ast[2]
come_froms = jump_cf_pop[0]
for come_from in come_froms:
if come_from.attr < tokens[first].offset:
return True
come_froms = jump_cf_pop[2]
if come_froms == "COME_FROM":
if come_froms.attr < tokens[first].offset:
return True
pass
elif come_froms == "_come_froms":
for come_from in come_froms:
if come_from.attr < tokens[first].offset:
return True
return False
elif testfalse[1] == "jmp_false":
jmp_false = testfalse[1]
if last == len(tokens):
last -= 1
while (isinstance(tokens[first].offset, str) and first < last):
first += 1
if first == last:
return True
while (first < last and isinstance(tokens[last].offset, str)):
last -= 1
if rule[0] == "iflaststmtl":
return not (jmp_false[0].attr <= tokens[last].offset)
else:
jmp_false_target = jmp_false[0].attr
if tokens[first].offset > jmp_false_target:
return True
return (
(jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD")
pass
pass
pass
# elif lhs == "and":
# return tokens[last+1] == "JUMP_FORWARD"
pass
class Python30ParserSingle(Python30Parser, PythonParserSingle):
pass
if __name__ == '__main__':
# Check grammar
p = Python30Parser()
p.remove_rules_30()
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.0:
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
## FIXME: try this
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
import sys
if len(sys.argv) > 1:
from spark_parser.spark import rule2str
for rule in sorted(p.rule2name.items()):
print(rule2str(rule[0]))

2
uncompyle6/parsers/parse34.py
View File

@@ -72,7 +72,7 @@ if __name__ == '__main__':
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.4:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(

2
uncompyle6/parsers/parse35.py
View File

@@ -262,7 +262,7 @@ if __name__ == '__main__':
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.5:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(

2
uncompyle6/parsers/parse36.py
View File

@@ -440,7 +440,7 @@ if __name__ == '__main__':
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.6:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(

50
uncompyle6/parsers/parse37.py
View File

@@ -140,31 +140,6 @@ class Python37Parser(Python36Parser):
"""
def customize_grammar_rules(self, tokens, customize):
self.remove_rules("""
async_forelse_stmt ::= SETUP_LOOP expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_FORWARD COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_FALSE
POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
JUMP_ABSOLUTE END_FINALLY COME_FROM
for_block POP_BLOCK
else_suite COME_FROM_LOOP
stmt ::= async_for_stmt36
async_for_stmt36 ::= SETUP_LOOP expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
YIELD_FROM
store
POP_BLOCK JUMP_BACK COME_FROM_EXCEPT DUP_TOP
LOAD_GLOBAL COMPARE_OP POP_JUMP_IF_TRUE
END_FINALLY continues COME_FROM
POP_TOP POP_TOP POP_TOP POP_EXCEPT
POP_TOP POP_BLOCK
COME_FROM_LOOP
""")
super(Python37Parser, self).customize_grammar_rules(tokens, customize)
class Python37ParserSingle(Python37Parser, PythonParserSingle):
@@ -172,22 +147,33 @@ class Python37ParserSingle(Python37Parser, PythonParserSingle):
if __name__ == '__main__':
# Check grammar
# FIXME: DRY this with other parseXX.py routines
p = Python37Parser()
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.7:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
opcode_set = set(s.opc.opname).union(
set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
""".split()
)
)
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))
import sys
if len(sys.argv) > 1:
from spark_parser.spark import rule2str
for rule in sorted(p.rule2name.items()):
print(rule2str(rule[0]))

34
uncompyle6/parsers/parse38.py
View File

@@ -159,7 +159,7 @@ class Python38Parser(Python37Parser):
super(Python38Parser, self).__init__(debug_parser)
self.customized = {}
def customize_grammar_rules(self, tokens, customize):
def remove_rules_38(self):
self.remove_rules("""
stmt ::= async_for_stmt37
stmt ::= for
@@ -226,7 +226,10 @@ class Python38Parser(Python37Parser):
""")
def customize_grammar_rules(self, tokens, customize):
super(Python37Parser, self).customize_grammar_rules(tokens, customize)
self.remove_rules_38()
self.check_reduce['ifstmt'] = 'tokens'
self.check_reduce['whileTruestmt38'] = 'tokens'
@@ -234,6 +237,7 @@ class Python38Parser(Python37Parser):
invalid = super(Python38Parser,
self).reduce_is_invalid(rule, ast,
tokens, first, last)
self.remove_rules_38()
if invalid:
return invalid
if rule[0] == 'ifstmt':
@@ -264,24 +268,34 @@ class Python38Parser(Python37Parser):
class Python38ParserSingle(Python38Parser, PythonParserSingle):
pass
if __name__ == '__main__':
if __name__ == "__main__":
# Check grammar
# FIXME: DRY this with other parseXX.py routines
p = Python38Parser()
p.remove_rules_38()
p.check_grammar()
from uncompyle6 import PYTHON_VERSION, IS_PYPY
if PYTHON_VERSION == 3.8:
lhs, rhs, tokens, right_recursive = p.check_sets()
lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
from uncompyle6.scanner import get_scanner
s = get_scanner(PYTHON_VERSION, IS_PYPY)
opcode_set = set(s.opc.opname).union(set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
opcode_set = set(s.opc.opname).union(
set(
"""JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
LAMBDA_MARKER RETURN_LAST
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)
# print(sorted(p.rule2name.items()))
import sys
if len(sys.argv) > 1:
from spark_parser.spark import rule2str
for rule in sorted(p.rule2name.items()):
print(rule2str(rule[0]))

8
uncompyle6/parsers/treenode.py
View File

@@ -54,3 +54,11 @@ class SyntaxTree(spark_AST):
rv += "\n" + child
i += 1
return rv
def first_child(self):
if len(self) > 0:
child = self[0]
if not isinstance(child, SyntaxTree):
return child
return self[0].first_child()
return self

20
uncompyle6/scanners/scanner3.py
View File

@@ -265,21 +265,29 @@ class Scanner3(Scanner):
# RAISE_VARARGS then we have a "raise" statement
# else we have an "assert" statement.
if self.version == 3.0:
# There is a an implied JUMP_IF_TRUE that we are not testing for (yet?) here
# Like 2.6, 3.0 doesn't have POP_JUMP_IF... so we have
# to go through more machinations
assert_can_follow = inst.opname == "POP_TOP" and i + 1 < n
if assert_can_follow:
prev_inst = self.insts[i - 1]
assert_can_follow = (
prev_inst.opname in ("JUMP_IF_TRUE", "JUMP_IF_FALSE")
and i + 1 < n )
jump_if_inst = prev_inst
else:
assert_can_follow = (
inst.opname in ("POP_JUMP_IF_TRUE", "POP_JUMP_IF_FALSE")
and i + 1 < n
)
jump_if_inst = inst
if assert_can_follow:
next_inst = self.insts[i + 1]
if (
next_inst.opname == "LOAD_GLOBAL"
and next_inst.argval == "AssertionError"
and inst.argval
and jump_if_inst.argval
):
raise_idx = self.offset2inst_index[self.prev_op[inst.argval]]
raise_idx = self.offset2inst_index[self.prev_op[jump_if_inst.argval]]
raise_inst = self.insts[raise_idx]
if raise_inst.opname.startswith("RAISE_VARARGS"):
self.load_asserts.add(next_inst.offset)
@@ -482,6 +490,12 @@ class Scanner3(Scanner):
and self.insts[i + 1].opname == "JUMP_FORWARD"
)
if (self.version == 3.0 and self.insts[i + 1].opname == "JUMP_FORWARD"
and not is_continue):
target_prev = self.offset2inst_index[self.prev_op[target]]
is_continue = (
self.insts[target_prev].opname == "SETUP_LOOP")
if is_continue or (
inst.offset in self.stmts
and (

1
uncompyle6/semantics/customize26_27.py
View File

@@ -40,6 +40,7 @@ def customize_for_version26_27(self, version):
'testtrue_then': ( 'not %p', (0, 22) ),
})
# FIXME: this should be a transformation
def n_call(node):
mapping = self._get_mapping(node)
key = node

26
uncompyle6/semantics/customize3.py
View File

@@ -181,6 +181,19 @@ def customize_for_version3(self, version):
# the iteration variable. These rules we can ignore
# since we pick up the iteration variable some other way and
# we definitely don't include in the source _[dd].
TABLE_DIRECT.update({
"ifstmt30": ( "%|if %c:\n%+%c%-",
(0, "testfalse_then"),
(1, "_ifstmts_jump30") ),
"ifnotstmt30": ( "%|if not %c:\n%+%c%-",
(0, "testtrue_then"),
(1, "_ifstmts_jump30") ),
"try_except30": ( "%|try:\n%+%c%-%c\n\n",
(1, "suite_stmts_opt"),
(4, "except_handler") ),
})
def n_comp_iter(node):
if node[0] == "expr":
n = node[0][0]
@@ -197,11 +210,14 @@ def customize_for_version3(self, version):
# FIXME: perhaps this can be folded into the 3.4+ case?
def n_yield_from(node):
assert node[0] == "expr"
assert node[0][0] == "get_iter"
# Skip over yield_from.expr.get_iter which adds an
# extra iter(). Maybe we can do in tranformation phase instead?
template = ("yield from %c", (0, "expr"))
self.template_engine(template, node[0][0])
if node[0][0] == "get_iter":
# Skip over yield_from.expr.get_iter which adds an
# extra iter(). Maybe we can do in tranformation phase instead?
template = ("yield from %c", (0, "expr"))
self.template_engine(template, node[0][0])
else:
template = ("yield from %c", (0, "attribute"))
self.template_engine(template, node[0][0][0])
self.prune()
self.n_yield_from = n_yield_from

2
uncompyle6/semantics/fragments.py
View File

@@ -792,7 +792,7 @@ class FragmentsWalker(pysource.SourceWalker, object):
self.name = code_name
# Issue created with later Python code generation is that there
# is a lamda set up with a dummy argument name that is then called
# is a lambda set up with a dummy argument name that is then called
# So we can't just translate that as is but need to replace the
# dummy name. Below we are picking out the variable name as seen
# in the code. And trying to generate code for the other parts

13
uncompyle6/semantics/transform.py
View File

@@ -115,11 +115,14 @@ class TreeTransform(GenericASTTraversal, object):
call = expr[0]
LOAD_ASSERT = call[0]
expr = call[1][0]
node = SyntaxTree(
kind,
[assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1]
)
if isinstance(call[1], SyntaxTree):
expr = call[1][0]
node = SyntaxTree(
kind,
[assert_expr, jump_cond, LOAD_ASSERT, expr, RAISE_VARARGS_1]
)
pass
pass
else:
# ifstmt
# 0. testexpr (2)