diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 0793c448..0f2a61e7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -4,11 +4,11 @@ about: Tell us about uncompyle6 bugs --- -__Note:__ Have you read https://github.com/rocky/python-uncompyle6/blob/master/HOW-TO-REPORT-A-BUG.md ? + + ## Description -Replace this text with a clear and concise description of the bug. + ## How to Reproduce -Please show both the input you gave and the + + ## Expected behavior -A clear and concise description of what you expected to happen. + ## Environment -_This section sometimes is optional but helpful to us._ + + ## Additional Environment or Context -_This section is optional._ + diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index cab53e3b..3cbb75be 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -6,16 +6,17 @@ about: Tell us about a new feature that you would like to see in uncompyle6 ## Description -Replace this text with a short description of the feature. This might -include same input and output. + ## Background -Replace this text with any additional background for the -feature, for example: user scenarios, or the value of the feature. + ## Tests -_This section is optional._ + diff --git a/test/bytecode_1.3/test_grammar.pyc b/test/bytecode_1.3/test_grammar.pyc deleted file mode 100644 index bb1e6eae..00000000 Binary files a/test/bytecode_1.3/test_grammar.pyc and /dev/null differ diff --git a/test/simple_source/expression/06_huge_list.py b/test/simple_source/expression/06_huge_list.py index 6dcd97d0..c1d8b5f2 100644 --- a/test/simple_source/expression/06_huge_list.py +++ b/test/simple_source/expression/06_huge_list.py @@ -1,4 +1,8 @@ # We have more than 1<<16 elements +# In Python2 this causes an EXTENDED_ARG instruction to be emitted and then we can check +# handling that. + +# It also triggers the of special rules for expr32 and expr1024 [ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,11,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index 545a0204..e226c607 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -44,6 +44,11 @@ class PythonParser(GenericASTBuilder): def __init__(self, SyntaxTree, start, debug): super(PythonParser, self).__init__(SyntaxTree, start, debug) # FIXME: customize per python parser version + + # These are the non-terminal we should collect into a list. + # For example instead of: + # stmts -> stmts stmt -> stmts stmt stmt ... + # collect as stmts -> stmt stmt ... nt_list = [ 'stmts', 'except_stmts', '_stmts', 'attributes', 'exprlist', 'kvlist', 'kwargs', 'come_froms', '_come_froms', diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 33985ac7..3aed264b 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015-2017 Rocky Bernstein +# Copyright (c) 2015-2018 Rocky Bernstein # Copyright (c) 2000-2002 by hartmut Goebel # # Copyright (c) 1999 John Aycock @@ -294,8 +294,20 @@ class Python2Parser(PythonParser): # The order of opname listed is roughly sorted below if opname_base in ('BUILD_LIST', 'BUILD_SET', 'BUILD_TUPLE'): + # We do this complicated test to speed up parsing of + # pathelogically long literals, especially those over 1024. + build_count = token.attr + thousands = (build_count//1024) + thirty32s = ((build_count//32) % 32) + if thirty32s > 0: + rule = "expr32 ::=%s" % (' expr' * 32) + self.add_unique_rule(rule, opname_base, build_count, customize) + if thousands > 0: + self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), + opname_base, build_count, customize) collection = opname_base[opname_base.find('_')+1:].lower() - rule = '%s ::= %s%s' % (collection, (token.attr * 'expr '), opname) + rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + + 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) self.add_unique_rules([ "expr ::= %s" % collection, rule], customize) diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index 941743ea..17b4cb8a 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -709,11 +709,24 @@ class Python3Parser(PythonParser): rule = ('load_closure ::= %s%s' % (('LOAD_CLOSURE ' * v), opname)) self.add_unique_rule(rule, opname, token.attr, customize) if not is_LOAD_CLOSURE or v == 0: + build_count = token.attr + thousands = (build_count//1024) + thirty32s = ((build_count//32) % 32) + if thirty32s > 0: + rule = "expr32 ::=%s" % (' expr' * 32) + self.add_unique_rule(rule, opname_base, build_count, customize) + pass + if thousands > 0: + self.add_unique_rule("expr1024 ::=%s" % (' expr32' * 32), + opname_base, build_count, customize) + pass collection = opname_base[opname_base.find('_')+1:].lower() - rule = '%s ::= %s%s' % (collection, 'expr ' * v, opname) + rule = (('%s ::= ' % collection) + 'expr1024 '*thousands + + 'expr32 '*thirty32s + 'expr '*(build_count % 32) + opname) self.add_unique_rules([ - 'expr ::= %s' % collection, + "expr ::= %s" % collection, rule], customize) + continue continue elif opname_base == 'BUILD_SLICE': if token.attr == 2: diff --git a/uncompyle6/scanners/scanner2.py b/uncompyle6/scanners/scanner2.py index 77156433..8de9dd54 100644 --- a/uncompyle6/scanners/scanner2.py +++ b/uncompyle6/scanners/scanner2.py @@ -110,6 +110,17 @@ class Scanner2(Scanner): + @staticmethod + def extended_arg_val(arg): + """Return integer value of an EXTENDED_ARG operand. + In Python2 this always the operand value shifted 16 bits since + the operand is always 2 bytes. In Python 3.6+ this changes to one byte. + """ + if PYTHON3: + return (arg << 16) + else: + return (arg << long(16)) + @staticmethod def unmangle_name(name, classname): """Remove __ from the end of _name_ if it starts with __classname__ diff --git a/uncompyle6/semantics/helper.py b/uncompyle6/semantics/helper.py index f5e10374..19b89770 100644 --- a/uncompyle6/semantics/helper.py +++ b/uncompyle6/semantics/helper.py @@ -148,10 +148,12 @@ def flatten_list(node): for elem in node: if elem == 'expr1024': for subelem in elem: + assert subelem == 'expr32' for subsubelem in subelem: flat_elems.append(subsubelem) elif elem == 'expr32': for subelem in elem: + assert subelem == 'expr' flat_elems.append(subelem) else: flat_elems.append(elem)