Merge branch 'master' into python-2.4

This commit is contained in:
rocky
2019-01-26 18:33:39 -05:00
30 changed files with 256 additions and 56 deletions

View File

@@ -1,10 +1,13 @@
language: python
sudo: false
python:
- '2.7' # this is a cheat here because travis doesn't do 2.4-2.6
matrix:
include:
- python: '3.7'
dist: xenial # required for Python >= 3.7 (travis-ci/travis-ci#9069)
install:
- pip install -e .
- pip install -r requirements-dev.txt

View File

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
echo "This script should be *sourced* rather than run directly through bash"
exit 1
fi
export PYVERSIONS='3.5.5 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'
export PYVERSIONS='3.2.6 3.6.8 3.7.2 2.6.9 3.3.7 2.7.15 3.2.6 3.1.5 3.4.8'

View File

@@ -7,7 +7,7 @@ def test_grammar():
def check_tokens(tokens, opcode_set):
remain_tokens = set(tokens) - opcode_set
remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub(r'_\d+$','', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
assert remain_tokens == set([]), \
@@ -46,6 +46,7 @@ def test_grammar():
unused_rhs.add("mkfunc_annotate")
unused_rhs.add("dict_comp")
unused_rhs.add("classdefdeco1")
unused_rhs.add("tryelsestmtl")
if PYTHON_VERSION >= 3.5:
expect_right_recursive.add((('l_stmts',
('lastl_stmt', 'come_froms', 'l_stmts'))))

View File

@@ -64,10 +64,12 @@ check-3.5: check-bytecode
#: Run working tests from Python 3.6
check-3.6: check-bytecode
$(PYTHON) test_pythonlib.py --bytecode-3.6 --weak-verify $(COMPILE)
$(PYTHON) test_pythonlib.py --bytecode-3.6-run --verify-run
#: Run working tests from Python 3.7
check-3.7: check-bytecode
$(PYTHON) test_pythonlib.py --bytecode-3.7 --weak-verify $(COMPILE)
$(PYTHON) test_pythonlib.py --bytecode-3.7-run --verify-run
# FIXME
#: this is called when running under pypy3.5-5.8.0 or pypy2-5.6.0
@@ -89,8 +91,9 @@ check-bytecode-2:
#: Check deparsing bytecode 3.x only
check-bytecode-3:
$(PYTHON) test_pythonlib.py --bytecode-3.0 \
--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2
--bytecode-3.1 --bytecode-3.2 --bytecode-3.3 \
--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-3.7 \
--bytecode-pypy3.2
#: Check deparsing on selected bytecode 3.x
check-bytecode-3-short:

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,10 @@
# Python 2.4 (and before?) bug in handling unconditional "else if true"
# Doesn't occur in Python > 2.4
# From Issue #187
def unconditional_if_true_24(foo):
if not foo:
pass
elif 1:
pass
else:
return None

View File

@@ -0,0 +1,22 @@
# Bug in Python 2.7 is code creating a (useless) JUMP_ABSOLUTE to the instruction right after
# the "raise" which causes the
# RUNNABLE!
def testit(a, b):
if a:
if not b:
raise AssertionError("test JUMP_ABSOLUTE to next instruction")
def testit2(a, b):
if a:
if not b:
raise AssertionError("test with dead code after raise")
x = 10
testit(False, True)
testit(False, False)
testit(True, True)
testit2(False, True)
testit2(False, False)
testit2(True, True)

View File

@@ -0,0 +1,24 @@
# Bug based on 2.7 test_itertools.py but mis-decompiled in Python 3.x bytecode
# The bug is confusing "try/else" with "try" as a result of the loop which causes
# the end of the except to jump back to the beginning of the loop, outside of the
# try. In 3.x we not distinguising this jump out of the loop with a jump to the
# end of the "try".
# RUNNABLE!
def testit(stmts):
# Bug was confusing When the except jumps back to the beginning of the block
# to the beginning of this for loop
x = 1
results = []
for stmt in stmts:
try:
x = eval(stmt)
except SyntaxError:
results.append(1)
else:
results.append(x)
return results
results = testit(["1 + 2", "1 +"])
assert results == [3, 1], "try with else failed"

View File

@@ -0,0 +1,15 @@
# Python 3 bug in not detecting the end bounds of if elif.
def testit(b):
if b == 1:
a = 1
elif b == 2:
a = 2
else:
a = 4
return a
for x in (1, 2, 4):
x = testit(x)
assert x is not None, "Should have returned a value, not None"
assert x == x

View File

@@ -0,0 +1,8 @@
# The bug in python 3.6+ was in parsing that we
# add END_IF_THEN and using that inside "return results"
def whcms_license_info(md5hash, datahash, results):
if md5hash == datahash:
try:
return md5hash
except:
return results

View File

@@ -0,0 +1,9 @@
# Bug in Python 3.6 and 3.7 was getting comma before **kw
def fn(arg, *, kwarg='test', **kw):
assert arg == 1
assert kwarg == 'testing'
assert kw['foo'] == 'bar'
fn(1, kwarg='testing', foo='bar')

View File

@@ -0,0 +1,19 @@
# From Python 3.7 pickle.py
# Bug was different code generation for chained comparisons than prior Python versions
def chained_compare_a(protocol):
if not 0 <= protocol <= 7:
raise ValueError("pickle protocol must be <= %d" % 7)
def chained_compare_b(a, obj):
if a:
if -0x80000000 <= obj <= 0x7fffffff:
return 5
chained_compare_a(3)
try:
chained_compare_a(8)
except ValueError:
pass
chained_compare_b(True, 0x0)

View File

@@ -323,10 +323,19 @@ def main(in_base, out_base, files, codes, outfile=None,
sys.stdout.write("%s\r" %
status_msg(do_verify, tot_files, okay_files, failed_files,
verify_failed_files, do_verify))
sys.stdout.flush()
try:
# FIXME: Something is weird with Pypy here
sys.stdout.flush()
except:
pass
if current_outfile:
sys.stdout.write("\n")
sys.stdout.flush()
try:
# FIXME: Something is weird with Pypy here
sys.stdout.flush()
except:
pass
pass
return (tot_files, okay_files, failed_files, verify_failed_files)

View File

@@ -531,14 +531,11 @@ class Python2Parser(PythonParser):
# Dead code testing...
# if lhs == 'while1elsestmt':
# from trepan.api import debug; debug()
if lhs in ('aug_assign1', 'aug_assign2') and ast[0] and ast[0][0] in ('and', 'or'):
return True
elif lhs in ('raise_stmt1',):
# We will assme 'LOAD_ASSERT' will be handled by an assert grammar rule
return (tokens[first] == 'LOAD_ASSERT' and
(last >= len(tokens) or tokens[last] not in
('COME_FROM', 'JUMP_BACK','JUMP_FORWARD')))
# We will assume 'LOAD_ASSERT' will be handled by an assert grammar rule
return (tokens[first] == 'LOAD_ASSERT' and (last >= len(tokens)))
elif rule == ('or', ('expr', 'jmp_true', 'expr', '\\e_come_from_opt')):
expr2 = ast[2]
return expr2 == 'expr' and expr2[0] == 'LOAD_ASSERT'

View File

@@ -93,15 +93,17 @@ class Python25Parser(Python26Parser):
super(Python25Parser, self).customize_grammar_rules(tokens, customize)
if self.version == 2.5:
self.check_reduce['try_except'] = 'tokens'
self.check_reduce['aug_assign1'] = 'AST'
## Don't need this for 2.5 yet..
# def reduce_is_invalid(self, rule, ast, tokens, first, last):
# invalid = super(Python25Parser,
# self).reduce_is_invalid(rule, ast,
# tokens, first, last)
# if invalid or tokens is None:
# return invalid
# return False
def reduce_is_invalid(self, rule, ast, tokens, first, last):
invalid = super(Python25Parser,
self).reduce_is_invalid(rule, ast,
tokens, first, last)
if invalid or tokens is None:
return invalid
if rule == ('aug_assign1', ('expr', 'expr', 'inplace_op', 'store')):
return ast[0][0] == 'and'
return False
class Python25ParserSingle(Python26Parser, PythonParserSingle):

View File

@@ -1,9 +1,10 @@
# Copyright (c) 2016-2018 Rocky Bernstein
# Copyright (c) 2016-2019 Rocky Bernstein
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
# Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
from uncompyle6.parser import PythonParserSingle
from xdis import next_offset
from uncompyle6.parser import PythonParserSingle, nop_func
from uncompyle6.parsers.parse2 import Python2Parser
class Python27Parser(Python2Parser):
@@ -196,6 +197,13 @@ class Python27Parser(Python2Parser):
POP_BLOCK LOAD_CONST COME_FROM suite_stmts_opt
END_FINALLY
""")
if 'PyPy' in customize:
# PyPy-specific customizations
self.addRule("""
return_if_stmt ::= ret_expr RETURN_END_IF come_froms
""", nop_func)
super(Python27Parser, self).customize_grammar_rules(tokens, customize)
self.check_reduce['and'] = 'AST'
# self.check_reduce['or'] = 'AST'
@@ -220,6 +228,12 @@ class Python27Parser(Python2Parser):
tokens[last].pattr == jmp_false.pattr)
elif rule[0] == ('raise_stmt1'):
return ast[0] == 'expr' and ast[0][0] == 'or'
elif rule[0] in ('assert', 'assert2'):
jump_inst = ast[1][0]
jump_target = jump_inst.attr
return not (last >= len(tokens)
or jump_target == tokens[last].offset
or jump_target == next_offset(ast[-1].op, ast[-1].opc, ast[-1].offset))
elif rule == ('list_if_not', ('expr', 'jmp_true', 'list_iter')):
jump_inst = ast[1][0]
jump_offset = jump_inst.attr

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2018 Rocky Bernstein
# Copyright (c) 2015-2019 Rocky Bernstein
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
# Copyright (c) 1999 John Aycock
@@ -26,6 +26,7 @@ If we succeed in creating a parse tree, then we have a Python program
that a later phase can turn into a sequence of ASCII text.
"""
from uncompyle6.scanners.tok import Token
from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
from uncompyle6.parsers.treenode import SyntaxTree
from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
@@ -185,23 +186,11 @@ class Python3Parser(PythonParser):
# one COME_FROM for Python 2.7 seems to associate the
# COME_FROM targets from the wrong places
try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler opt_come_from_except
# this is nested inside a try_except
tryfinallystmt ::= SETUP_FINALLY suite_stmts_opt
POP_BLOCK LOAD_CONST
COME_FROM_FINALLY suite_stmts_opt END_FINALLY
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suite come_from_except_clauses
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suite come_froms
tryelsestmtl ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suitel come_from_except_clauses
except_handler ::= jmp_abs COME_FROM except_stmts
END_FINALLY
except_handler ::= jmp_abs COME_FROM_EXCEPT except_stmts
@@ -564,9 +553,10 @@ class Python3Parser(PythonParser):
# include instructions that don't need customization,
# but we'll do a finer check after the rough breakout.
customize_instruction_basenames = frozenset(
('BUILD', 'CALL', 'CONTINUE', 'DELETE', 'GET',
'JUMP', 'LOAD', 'LOOKUP', 'MAKE',
'RETURN', 'RAISE', 'UNPACK'))
('BUILD', 'CALL', 'CONTINUE', 'DELETE', 'GET',
'JUMP', 'LOAD', 'LOOKUP', 'MAKE',
'RETURN', 'RAISE', 'SETUP',
'UNPACK'))
# Opcode names in the custom_ops_processed set have rules that get added
# unconditionally and the rules are constant. So they need to be done
@@ -1119,6 +1109,26 @@ class Python3Parser(PythonParser):
raise_stmt2 ::= expr expr RAISE_VARARGS_2
""", nop_func)
custom_ops_processed.add(opname)
elif opname == 'SETUP_EXCEPT':
self.addRule("""
try_except ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler opt_come_from_except
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suite come_from_except_clauses
tryelsestmt ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suite come_froms
tryelsestmtl ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler else_suitel come_from_except_clauses
stmt ::= tryelsestmtl3
tryelsestmtl3 ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
except_handler COME_FROM else_suitel
opt_come_from_except
""", nop_func)
custom_ops_processed.add(opname)
elif opname_base in ('UNPACK_EX',):
before_count, after_count = token.attr
rule = 'unpack ::= ' + opname + ' store' * (before_count + after_count + 1)
@@ -1137,8 +1147,11 @@ class Python3Parser(PythonParser):
self.check_reduce['aug_assign2'] = 'AST'
self.check_reduce['while1stmt'] = 'noAST'
self.check_reduce['while1elsestmt'] = 'noAST'
self.check_reduce['ifelsestmt'] = 'AST'
self.check_reduce['annotate_tuple'] = 'noAST'
self.check_reduce['kwarg'] = 'noAST'
self.check_reduce['try_except'] = 'AST'
# FIXME: remove parser errors caused by the below
# self.check_reduce['while1elsestmt'] = 'noAST'
return
@@ -1180,6 +1193,16 @@ class Python3Parser(PythonParser):
if last == n:
return False
return tokens[first].attr > tokens[last].offset
elif rule == ('try_except',
('SETUP_EXCEPT', 'suite_stmts_opt', 'POP_BLOCK',
'except_handler', 'opt_come_from_except')):
come_from_except = ast[-1]
if come_from_except[0] == 'COME_FROM':
# There should be at last two COME_FROMs, one from an
# exception handler and one from the try. Otherwise
# we have a try/else.
return True
pass
elif lhs == 'while1stmt':
# If there is a fall through to the COME_FROM_LOOP. then this is
@@ -1213,6 +1236,14 @@ class Python3Parser(PythonParser):
if offset != tokens[first].attr:
return True
return False
elif rule == ('ifelsestmt',
('testexpr', 'c_stmts_opt', 'jump_forward_else', 'else_suite', '_come_froms')):
# Make sure the highest/smallest "come from" offset comes inside the "if".
come_froms = ast[-1]
if not isinstance(come_froms, Token):
return tokens[first].offset > come_froms[-1].attr
return False
return False
class Python30Parser(Python3Parser):

View File

@@ -84,7 +84,7 @@ if __name__ == '__main__':
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)

View File

@@ -266,7 +266,7 @@ if __name__ == '__main__':
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2016-2018 Rocky Bernstein
# Copyright (c) 2016-2019 Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -121,6 +121,7 @@ class Python36Parser(Python35Parser):
# """)
super(Python36Parser, self).customize_grammar_rules(tokens, customize)
self.remove_rules("""
except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM
async_for_stmt ::= SETUP_LOOP expr
GET_AITER
LOAD_CONST YIELD_FROM SETUP_EXCEPT GET_ANEXT LOAD_CONST
@@ -237,6 +238,8 @@ class Python36Parser(Python35Parser):
WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
"""
self.addRule(rules_str, nop_func)
pass
pass
def custom_classfunc_rule(self, opname, token, customize, next_token):
@@ -343,7 +346,8 @@ class Python36Parser(Python35Parser):
if nt[0] == 'call_kw':
return True
nt = nt[0]
pass
pass
return False
class Python36ParserSingle(Python36Parser, PythonParserSingle):
pass
@@ -364,7 +368,7 @@ if __name__ == '__main__':
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)

View File

@@ -66,6 +66,17 @@ class Python37Parser(Python36Parser):
# FIXME: generalize and specialize
call ::= expr CALL_METHOD_0
testtrue ::= compare_chained37
compare_chained37 ::= expr compare_chained1a_37
compare_chained37 ::= expr compare_chained1b_37
compare_chained2a_37 ::= expr COMPARE_OP POP_JUMP_IF_TRUE JUMP_FORWARD
compare_chained2b_37 ::= expr COMPARE_OP POP_JUMP_IF_FALSE JUMP_FORWARD
compare_chained1a_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained2a_37 ELSE POP_TOP COME_FROM
compare_chained1b_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
compare_chained2b_37 ELSE POP_TOP JUMP_FORWARD COME_FROM
"""
def customize_grammar_rules(self, tokens, customize):
@@ -103,7 +114,7 @@ if __name__ == '__main__':
""".split()))
remain_tokens = set(tokens) - opcode_set
import re
remain_tokens = set([re.sub('_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
remain_tokens = set(remain_tokens) - opcode_set
print(remain_tokens)

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2018 by Rocky Bernstein
# Copyright (c) 2015-2019 by Rocky Bernstein
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
#
@@ -886,9 +886,10 @@ class Scanner2(Scanner):
# or a conditional assignment like:
# x = 1 if x else 2
#
# There are other contexts we may need to consider
# like whether the target is "END_FINALLY"
# or if the condition jump is to a forward location
# There are other situations we may need to consider, like
# if the condition jump is to a forward location.
# Also the existence of a jump to the instruction after "END_FINALLY"
# will distinguish "try/else" from "try".
code_pre_rtarget = code[pre_rtarget]
if code_pre_rtarget in self.jump_forward:

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2015-2018 by Rocky Bernstein
# Copyright (c) 2015-2019 by Rocky Bernstein
# Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
# Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
#
@@ -859,9 +859,10 @@ class Scanner3(Scanner):
# For 3.5, in addition the JUMP_FORWARD above we could have
# JUMP_BACK or CONTINUE
#
# There are other contexts we may need to consider
# like whether the target is "END_FINALLY"
# or if the condition jump is to a forward location
# There are other situations we may need to consider, like
# if the condition jump is to a forward location.
# Also the existence of a jump to the instruction after "END_FINALLY"
# will distinguish "try/else" from "try".
if self.is_jump_forward(pre_rtarget) or (rtarget_is_ja and self.version >= 3.5):
if_end = self.get_target(pre_rtarget)

View File

@@ -113,7 +113,7 @@ class Token: # Python 2.4 can't have empty ()
pattr = self.opc.cmp_op[self.attr]
# And so on. See xdis/bytecode.py get_instructions_bytes
pass
elif re.search('_\d+$', self.kind):
elif re.search(r'_\d+$', self.kind):
return "%s%s%s" % (prefix, offset_opname, argstr)
else:
pattr = ''

View File

@@ -1,4 +1,4 @@
# Copyright (c) 2018 by Rocky Bernstein
# Copyright (c) 2018-2019 by Rocky Bernstein
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -263,6 +263,12 @@ def customize_for_version3(self, version):
self.prune() # stop recursing
self.n_mkfunc_annotate = n_mkfunc_annotate
TABLE_DIRECT.update({
'tryelsestmtl3': ( '%|try:\n%+%c%-%c%|else:\n%+%c%-',
(1, 'suite_stmts_opt'),
(3, 'except_handler'),
(5, 'else_suitel') ),
})
if version >= 3.4:
########################
# Python 3.4+ Additions
@@ -911,6 +917,16 @@ def customize_for_version3(self, version):
'async_for_stmt': (
'%|async for %c in %c:\n%+%c%-%-\n\n',
7, 1, 17),
'compare_chained1a_37': ( ' %[3]{pattr.replace("-", " ")} %p %p',
(0, 19),
(-4, 19)),
'compare_chained1b_37': ( ' %[3]{pattr.replace("-", " ")} %p %p',
(0, 19),
(-5, 19)),
'compare_chained2a_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)),
'compare_chained2b_37': ( '%[1]{pattr.replace("-", " ")} %p', (0, 19)),
})
pass
pass # version >= 3.6

View File

@@ -739,7 +739,7 @@ def make_function3(self, node, is_lambda, nested=1, code_node=None):
self.write(sep)
self.write("%s=%s" % (n, defaults[i]))
sep = ', '
ends_in_comma = True
ends_in_comma = False
pass
pass
pass