diff --git a/Makefile b/Makefile index 027e14a1..910e17d6 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ check-2.7 check-3.3 check-3.4: pytest #: Tests for Python 3.2 and 3.5 - pytest doesn't work here # Or rather 3.5 doesn't work not on Travis -check-3.2 check-3.5: +check-3.2 check-3.5 check-3.6: $(MAKE) -C test $@ #:Tests for Python 2.6 (doesn't have pytest) diff --git a/README.rst b/README.rst index 2de6e486..b0a6f19a 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ Introduction ------------ *uncompyle6* translates Python bytecode back into equivalent Python -source code. It accepts bytecodes from Python version 2.3 to 3.5 or +source code. It accepts bytecodes from Python version 2.3 to 3.6 or so, including PyPy bytecode. Why this? @@ -45,7 +45,7 @@ Requirements This project requires Python 2.6 or later, PyPy 3-2.4, or PyPy-5.0.1. The bytecode files it can read has been tested on Python bytecodes from -versions 2.3-2.7, and 3.2-3.5 and the above-mentioned PyPy versions. +versions 2.3-2.7, and 3.2-3.6 and the above-mentioned PyPy versions. Installation ------------ diff --git a/test/Makefile b/test/Makefile index 51f9f372..d4db0489 100644 --- a/test/Makefile +++ b/test/Makefile @@ -38,6 +38,10 @@ check-3.4: check-bytecode check-3.4-ok check-2.7-ok check-3.5: check-bytecode $(PYTHON) test_pythonlib.py --bytecode-3.5 --verify $(COMPILE) +#: Run working tests from Python 3.6 +check-3.6: check-bytecode + $(PYTHON) test_pythonlib.py --bytecode-3.6 --verify $(COMPILE) + #: Check deparsing only, but from a different Python version check-disasm: $(PYTHON) dis-compare.py @@ -50,7 +54,7 @@ check-bytecode-2: #: Check deparsing bytecode 3.x only check-bytecode-3: $(PYTHON) test_pythonlib.py --bytecode-3.2 --bytecode-3.3 \ - --bytecode-3.4 --bytecode-3.5 --bytecode-pypy3.2 + --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 --bytecode-pypy3.2 #: Check deparsing bytecode that works running Python 2 and Python 3 check-bytecode: check-bytecode-3 @@ -93,6 +97,10 @@ check-bytecode-3.4: check-bytecode-3.5: $(PYTHON) test_pythonlib.py --bytecode-3.5 +#: Check deparsing Python 3.6 +check-bytecode-3.6: + $(PYTHON) test_pythonlib.py --bytecode-3.6 + #: short tests for bytecodes only for this version of Python check-native-short: $(PYTHON) test_pythonlib.py --bytecode-$(PYTHON_VERSION) --verify $(COMPILE) diff --git a/test/test_pythonlib.py b/test/test_pythonlib.py index 801b23a5..ea98f7e1 100755 --- a/test/test_pythonlib.py +++ b/test/test_pythonlib.py @@ -72,13 +72,13 @@ test_options = { PYOC, 'base_2.7', 2.7), } -for vers in (2.7, 3.4, 3.5): +for vers in (2.7, 3.4, 3.5, 3.6): pythonlib = "ok_lib%s" % vers key = "ok-%s" % vers test_options[key] = (os.path.join(src_dir, pythonlib), PYOC, key, vers) pass -for vers in (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 'pypy3.2', 'pypy2.7'): +for vers in (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 'pypy3.2', 'pypy2.7'): bytecode = "bytecode_%s" % vers key = "bytecode-%s" % vers test_options[key] = (bytecode, PYC, bytecode, vers) diff --git a/uncompyle6/bin/uncompile.py b/uncompyle6/bin/uncompile.py index 2d9b4031..7e2a2cbe 100755 --- a/uncompyle6/bin/uncompile.py +++ b/uncompyle6/bin/uncompile.py @@ -64,8 +64,8 @@ def usage(): def main_bin(): - if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 2), (3, 3), (3, 4), (3, 5))): - print('Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4 or 3.5' % program, + if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6))): + print('Error: %s requires Python 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, or 3.6' % program, file=sys.stderr) sys.exit(-1) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index a51ad1b9..7d70e9d8 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -365,7 +365,7 @@ class Python3Parser(PythonParser): call_function ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc {expr}^n-1 CALL_FUNCTION_n - """ + """ # Low byte indicates number of positional paramters, # high byte number of positional parameters args_pos = token.attr & 0xff @@ -444,7 +444,8 @@ class Python3Parser(PythonParser): For PYPY: load_attr ::= expr LOOKUP_METHOD call_function ::= expr CALL_METHOD - """ + """ + saw_format_value = False for i, token in enumerate(tokens): opname = token.type opname_base = opname[:opname.rfind('_')] @@ -457,8 +458,19 @@ class Python3Parser(PythonParser): assign2_pypy ::= expr expr designator designator """, nop_func) continue + elif opname == 'FORMAT_VALUE': + # Python 3.6+ + self.addRule(""" + formatted_value ::= LOAD_FAST FORMAT_VALUE + formatted_value ::= LOAD_NAME FORMAT_VALUE + str ::= LOAD_CONST + formatted_value_or_str ::= formatted_value + formatted_value_or_str ::= str + """, nop_func) + saw_format_value = True + elif opname in ('CALL_FUNCTION', 'CALL_FUNCTION_VAR', - 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): + 'CALL_FUNCTION_VAR_KW', 'CALL_FUNCTION_KW'): self.custom_classfunc_rule(opname, token, customize) elif opname == 'LOAD_DICTCOMP': rule_pat = ("dictcomp ::= LOAD_DICTCOMP %sMAKE_FUNCTION_0 expr " @@ -479,6 +491,15 @@ class Python3Parser(PythonParser): if opname_base == 'BUILD_TUPLE': rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname)) self.add_unique_rule(rule, opname, token.attr, customize) + if opname_base == 'BUILD_LIST' and saw_format_value: + format_or_str_n = "formatted_value_or_str_%s" % v + self.addRule(""" + expr ::= joined_str + joined_str ::= LOAD_CONST LOAD_ATTR %s CALL_FUNCTION_1 + %s ::= %s%s + """ % (format_or_str_n, format_or_str_n, ("formatted_value_or_str " *v), opname), + nop_func) + elif opname == 'LOOKUP_METHOD': # A PyPy speciality - DRY with parse2 self.add_unique_rule("load_attr ::= expr LOOKUP_METHOD", diff --git a/uncompyle6/parsers/parse36.py b/uncompyle6/parsers/parse36.py new file mode 100644 index 00000000..936fabd4 --- /dev/null +++ b/uncompyle6/parsers/parse36.py @@ -0,0 +1,52 @@ +# Copyright (c) 2016 Rocky Bernstein +""" +spark grammar differences over Python 3.5 for Python 3.6. +""" +from __future__ import print_function + +from uncompyle6.parser import PythonParserSingle +from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG +from uncompyle6.parsers.parse35 import Python35Parser + +class Python36Parser(Python35Parser): + + def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG): + super(Python36Parser, self).__init__(debug_parser) + self.customized = {} + + def p_36misc(self, args): + """ + formatted_value ::= LOAD_FAST FORMAT_VALUE + str ::= LOAD_CONST + joined_str ::= LOAD_CONST LOAD_ATTR format_value_or_strs + BUILD_LIST CALL_FUNCTION + format_value_or_strs ::= format_value_or_strs format_value_or_str + format_value_or_strs ::= format_value_or_str + format_value_or_str ::= format_value + format_value_or_str ::= str + """ + +class Python36ParserSingle(Python36Parser, PythonParserSingle): + pass + +if __name__ == '__main__': + # Check grammar + p = Python36Parser() + p.checkGrammar() + from uncompyle6 import PYTHON_VERSION, IS_PYPY + if PYTHON_VERSION == 3.6: + lhs, rhs, tokens, right_recursive = p.checkSets() + from uncompyle6.scanner import get_scanner + s = get_scanner(PYTHON_VERSION, IS_PYPY) + opcode_set = set(s.opc.opname).union(set( + """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM + LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME + LAMBDA_MARKER RETURN_LAST + """.split())) + remain_tokens = set(tokens) - opcode_set + import re + remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens]) + remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens]) + remain_tokens = set(remain_tokens) - opcode_set + print(remain_tokens) + # print(sorted(p.rule2name.items())) diff --git a/uncompyle6/scanner.py b/uncompyle6/scanner.py index 9e587beb..1a0d8758 100755 --- a/uncompyle6/scanner.py +++ b/uncompyle6/scanner.py @@ -22,7 +22,7 @@ from uncompyle6 import PYTHON3, IS_PYPY from uncompyle6.scanners.tok import Token # The byte code versions we support -PYTHON_VERSIONS = (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5) +PYTHON_VERSIONS = (2.3, 2.4, 2.5, 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6) # FIXME: DRY if PYTHON3: diff --git a/uncompyle6/scanners/scanner36.py b/uncompyle6/scanners/scanner36.py new file mode 100644 index 00000000..fe212aea --- /dev/null +++ b/uncompyle6/scanners/scanner36.py @@ -0,0 +1,35 @@ +# Copyright (c) 2016 by Rocky Bernstein +""" +Python 3.5 bytecode scanner/deparser + +This sets up opcodes Python's 3.5 and calls a generalized +scanner routine for Python 3. +""" + +from __future__ import print_function + +from uncompyle6.scanners.scanner3 import Scanner3 + +# bytecode verification, verify(), uses JUMP_OPs from here +from xdis.opcodes import opcode_36 as opc +JUMP_OPs = map(lambda op: opc.opname[op], opc.hasjrel + opc.hasjabs) + +class Scanner36(Scanner3): + + def __init__(self, show_asm=None): + Scanner3.__init__(self, 3.6, show_asm) + return + pass + +if __name__ == "__main__": + from uncompyle6 import PYTHON_VERSION + if PYTHON_VERSION == 3.6: + import inspect + co = inspect.currentframe().f_code + tokens, customize = Scanner36().disassemble(co) + for t in tokens: + print(t.format()) + pass + else: + print("Need to be Python 3.6 to demo; I am %s." % + PYTHON_VERSION) diff --git a/uncompyle6/semantics/aligner.py b/uncompyle6/semantics/aligner.py index 7e26b726..eb154062 100644 --- a/uncompyle6/semantics/aligner.py +++ b/uncompyle6/semantics/aligner.py @@ -18,8 +18,16 @@ class AligningWalker(SourceWalker, object): self.pending_newlines = max(self.pending_newlines, 1) def write(self, *data): + from trepan.api import debug; debug() + if (len(data) == 1) and data[0] == self.indent: + diff = max(self.pending_newlines, + self.desired_line_number - self.current_line_number) + self.f.write('\n'*diff) + self.current_line_number += diff + self.pending_newlines = 0 if (len(data) == 0) or (len(data) == 1 and data[0] == ''): return + out = ''.join((str(j) for j in data)) n = 0 for i in out: diff --git a/uncompyle6/semantics/pysource.py b/uncompyle6/semantics/pysource.py index 35fdd689..9a1c4731 100644 --- a/uncompyle6/semantics/pysource.py +++ b/uncompyle6/semantics/pysource.py @@ -586,6 +586,14 @@ class SourceWalker(GenericASTTraversal, object): TABLE_DIRECT.update({ 'LOAD_CLASSDEREF': ( '%{pattr}', ), }) + if version >= 3.6: + ######################## + # Python 3.6+ Additions + ####################### + TABLE_DIRECT.update({ + 'formatted_value': ( '{%c}', 0), + 'joined_str': ( "f'%c'", 2), + }) return f = property(lambda s: s.params['f'], @@ -836,6 +844,10 @@ class SourceWalker(GenericASTTraversal, object): self.prec += 1 self.prune() + def n_str(self, node): + self.write(node[0].pattr) + self.prune() + def n_LOAD_CONST(self, node): data = node.pattr; datatype = type(data) if isinstance(datatype, int) and data == minint: diff --git a/uncompyle6/verify.py b/uncompyle6/verify.py index e91aa1c9..d18719dd 100755 --- a/uncompyle6/verify.py +++ b/uncompyle6/verify.py @@ -215,6 +215,9 @@ def cmp_code_objects(version, is_pypy, code_obj1, code_obj2, name=''): elif version == 3.5: import uncompyle6.scanners.scanner35 as scan scanner = scan.Scanner35() + elif version == 3.6: + import uncompyle6.scanners.scanner36 as scan + scanner = scan.Scanner36() global JUMP_OPs JUMP_OPs = list(scan.JUMP_OPs) + ['JUMP_BACK']