3.0 tolerance

before tokenization -> disassembly
after tokenization -> tokenization

.circleci/config.yml

@@ -1,77 +0,0 @@
-version: 2
-filters:
-  branches:
-    only: master
-jobs:
-  build:
-    working_directory: ~/rocky/python-uncompyle6
-    parallelism: 1
-    shell: /bin/bash --login
-    # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
-    # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
-    environment:
-      CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
-      CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
-      COMPILE: --compile
-    # To see the list of pre-built images that CircleCI provides for most common languages see
-    # https://circleci.com/docs/2.0/circleci-images/
-    docker:
-      - image: circleci/python:3.6.9
-    steps:
-    # Machine Setup
-    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
-    - checkout
-    # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
-    # In many cases you can simplify this from what is generated here.
-    # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'
-    - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS
-    # This is based on your 1.0 configuration file or project settings
-    - run:
-        working_directory: ~/rocky/python-uncompyle6
-        command: pip install --user virtualenv && pip install --user nose && pip install --user pep8
-    # Dependencies
-    #   This would typically go in either a build or a build-and-test job when using workflows
-    # Restore the dependency cache
-    - restore_cache:
-        keys:
-          - v2-dependencies-{{ .Branch }}-
-          # fallback to using the latest cache if no exact match is found
-          - v2-dependencies-
-
-    - run:
-        command: | # Use pip to install dependengcies
-          pip install --user --upgrade setuptools
-          # Until the next release
-          pip install git+https://github.com/rocky/python-xdis#egg=xdis
-          pip install --user -e .
-          pip install --user -r requirements-dev.txt
-
-    # Save dependency cache
-    - save_cache:
-        key: v2-dependencies-{{ .Branch }}-{{ epoch }}
-        paths:
-        # This is a broad list of cache paths to include many possible development environments
-        # You can probably delete some of these entries
-        - vendor/bundle
-        - ~/virtualenvs
-        - ~/.m2
-        - ~/.ivy2
-        - ~/.bundle
-        - ~/.cache/bower
-
-    # Test
-    #   This would typically be a build job when using workflows, possibly combined with build
-    # This is based on your 1.0 configuration file or project settings
-    - run: sudo python ./setup.py develop && make check-3.6
-    - run: cd ./test/stdlib && bash ./runtests.sh 'test_[p-z]*.py'
-    # Teardown
-    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-    # Save test results
-    - store_test_results:
-        path: /tmp/circleci-test-results
-    # Save artifacts
-    - store_artifacts:
-        path: /tmp/circleci-artifacts
-    - store_artifacts:
-        path: /tmp/circleci-test-results

.github/FUNDING.yml

@@ -6,7 +6,7 @@ open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
-liberapay: # Replace with a single Liberapay username
+liberapay: rocky
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
 custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug-report.md

@@ -7,14 +7,19 @@ about: Tell us about uncompyle6 bugs
 <!-- __Note:__ If you are using this program to do something illegal - don't.
 The issue may be flagged to make it easier for those looking for illegal activity.
 
-Bugs are not for asking questions about a problem you
+If you are reporting a bug in decompilation, it will probably not be acted upon
+unless it is narrowed to a small example. You may have to do some work remove
+extraneous code from the source example. Most bugs can be expressed in 30 lines of
+code.
+
+Issues are not for asking questions about a problem you
 are trying to solve that involve the use of uncompyle6 along the way,
 although I may be more tolerant of this if you sponsor the project.
 
 Bugs are also not for general or novice kind help on how to install
-this Python program in your environment in the way you would like to
-have it set up, or how to interpret a Python traceback e.g. that winds
-up saying Python X.Y.Z is not supported.
+this Python program and its dependencies in your environment, or in
+the way you would like to have it set up, or how to interpret a Python
+traceback e.g. that winds up saying Python X.Y.Z is not supported.
 
 For these kinds of things, you will save yourself time by asking
 instead on forums like StackOverflow that are geared to helping people
@@ -50,8 +55,9 @@ Prerequisites/Caveats
 
 * Make sure the bytecode you have can be disassembled with a
   disassembler and produces valid results.
+* Try to make the bytecode that exhibits a bug as small as possible.
 * Don't put bytecode and corresponding source code on any service that
-  requires registration to download.
+  requires registration to download. Instead attach it as a zip file.
 * When you open a bug report there is no privacy. If you need privacy, then
   contact me by email and explain who you are and the need for privacy.
   But be mindful that you may be asked to sponsor the project for the
@@ -80,7 +86,7 @@ $ uncompyle6 <command-line-options>
 $
 ```
 
-Provide links to the Python bytecode. For example, you can create a
+Attach a zip file to the Python bytecode or a
 gist with the information. If you have the correct source code, you
 can add that too.
 
@@ -107,6 +113,7 @@ If this is too long, then try narrowing the problem to something short.
 Please modify for your setup
 
 - Uncompyle6 version: output from  `uncompyle6 --version` or `pip show uncompyle6`
+- xdis version: output from `pydisasm --version` or or `pip show xdis`
 - Python version for the version of Python the byte-compiled the file: `python -c "import sys; print(sys.version)"` where `python` is the correct CPython or PyPy binary.
 - OS and Version: [e.g. Ubuntu bionic]
 
@@ -118,7 +125,11 @@ Please modify for your setup
 
 ## Priority
 
-<!-- If this is blocking some important activity let us know what activity it blocks. -->
+<!-- If this is important for a particular public good state that here.
+     If this is blocking some important activity let us know what activity it blocks.
+
+	 Otherwise, we'll assume this has the lowest priority in addressing.
+	 -->
 
 ## Additional Context
 

.github/workflows/osx.yml

@@ -1,31 +0,0 @@
-name: uncompyle6 (osx)
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        os: [macOS]
-        python-version: [3.7, 3.8]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        # Until the next xdis release
-        pip install git+https://github.com/rocky/python-xdis#egg=xdis
-        pip install -e .
-        pip install -r requirements-dev.txt
-    - name: Test uncompyle6
-      run: |
-        make check

.github/workflows/ubuntu.yml

@@ -1,30 +0,0 @@
-name: uncompyle6 (ubuntu)
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.7, 3.8]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        # Until the next xdis release
-        pip install git+https://github.com/rocky/python-xdis#egg=xdis
-        pip install -e .
-        pip install -r requirements-dev.txt
-    - name: Test uncompyle6
-      run: |
-        make check

.github/workflows/windows.yml

@@ -1,31 +0,0 @@
-name: uncompyle6 (windows)
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        os: [windows]
-        python-version: [3.7, 3.8]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        # Until the next xdis release
-        pip install git+https://github.com/rocky/python-xdis#egg=xdis
-        pip install -e .
-        pip install -r requirements-dev.txt
-    - name: Test uncompyle6
-      run: |
-        make check

.gitignore

@@ -2,6 +2,7 @@
 *.pyo
 *_dis
 *~
+.mypy_cache
 /.cache
 /.eggs
 /.hypothesis
@@ -10,16 +11,17 @@
 /.pytest_cache
 /.python-version
 /.tox
-.mypy_cache
 /.venv*
 /README
 /__pkginfo__.pyc
 /dist
 /how-to-make-a-release.txt
 /nose-*.egg
+/pycharm-venv
 /tmp
 /uncompyle6.egg-info
 /unpyc
+/venv
 ChangeLog
 __pycache__
 build

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+default_language_version:
+  python: python
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    - id: check-merge-conflict
+    - id: debug-statements
+      stages: [commit]
+    - id: end-of-file-fixer
+      stages: [commit]

HISTORY.md

@@ -7,7 +7,7 @@ In the interest of shortening what is written here, I am going to start where we
 For the earlier history up to 2006 and the code up until Python 2.4, which I find interesting, look at that link.
 
 Sometime around 2014 was the dawn of ["uncompyle" and PyPI](https://pypi.python.org/pypi/uncompyle/1.1) — the era of
-public version control. Dan Pascu's code although not public used [darcs](http://darcs.net/) for version control. I converted the darcs to to git and put this at [decompyle-2.4](https://github.com/rocky/decompile-2.4).
+public version control. Dan Pascu's code although not public used [darcs](http://darcs.net/) for version control. I converted the darcs repository to git and put this at [decompyle-2.4](https://github.com/rocky/decompile-2.4).
 
 # uncompyle, unpyc
 
@@ -17,7 +17,7 @@ The project exists not only on [github](https://github.com/gstarnberger/uncompyl
 [bitbucket](https://bitbucket.org/gstarnberger/uncompyle) and later the defunct [google
 code](https://code.google.com/archive/p/unpyc/) under the name _unpyc_. The git/svn history goes back to 2009. Somewhere in there the name was changed from "decompyle" to "unpyc" by Keknehv, and then to "uncompyle" by Guenther Starnberger.
 
-The name Thomas Grainger isn't found in (m)any of the commits in the several years of active development. First Keknehv worked on this up to Python 2.5 or so while acceping Python bytecode back to 2.0 or so. Then "hamled" made a few commits earler on, while Eike Siewertsen made a few commits later on. But mostly "wibiti", and Guenther Starnberger got the code to where uncompyle2 was around 2012.
+The name Thomas Grainger isn't found in (m)any of the commits in the several years of active development. First Keknehv worked on this up to Python 2.5 or so while accepting Python bytecode back to 2.0 or so. Then "hamled" made a few commits earlier on, while Eike Siewertsen made a few commits later on. But mostly "wibiti", and Guenther Starnberger got the code to where uncompyle2 was around 2012.
 
 While John Aycock and Hartmut Goebel were well versed in compiler technology, those that have come afterwards don't seem to have been as facile in it.  Furthermore, documentation or guidance on how the decompiler code worked, comparison to a conventional compiler pipeline, how to add new constructs, or debug grammars was weak. Some of the grammar tracing and error reporting was a bit weak as well.
 
@@ -38,7 +38,7 @@ I started working on this late 2015, mostly to add fragment support. In that, I
 * this project - grammar and semantic actions for decompiling
   ([uncompyle6](https://pypi.python.org/pypi/uncompyle6)).
 
- `uncompyle6`, abandons the idea found in some 2.7 version of `uncompyle` that support Python 2.6 and 2.5 by trying to rewite opcodes at the bytecode level.
+ `uncompyle6`, abandons the idea found in some 2.7 version of `uncompyle` that support Python 2.6 and 2.5 by trying to rewrite opcodes at the bytecode level.
 
 Having a grammar per Python version is simpler to maintain, cleaner and it scales indefinitely.
 
@@ -68,13 +68,13 @@ project is largely by Michael Hansen and Darryl Pogue. If they supported getting
 
 # So you want to write a decompiler for Python?
 
-If you think, as I am sure will happen in the future, "hey, I can just write a decompiler from scratch and not have to deal with all all of the complexity in uncompyle6", think again. What is likely to happen is that you'll get at best a 90% solution working for a single Python release that will be obsolete in about a year, and more obsolete each subsequent year.
+If you think, as I am sure will happen in the future, "hey, I can just write a decompiler from scratch and not have to deal with all of the complexity in uncompyle6", think again. What is likely to happen is that you'll get at best a 90% solution working for a single Python release that will be obsolete in about a year, and more obsolete each subsequent year.
 
 Writing a decompiler for Python gets harder as it Python progresses. Writing decompiler for Python 3.7 isn't as easy as it was for Python 2.2. For one thing, now that Python has a well-established AST, that opens another interface by which code can be improved.
 
 In Python 3.10 I am seeing (for the first time?) bytecode getting moved around so that it is no longer the case that line numbers have to be strictly increasing as bytecode offsets increase. And I am seeing dead code appear as well.
 
-That said, if you still feel you want to write a single version decompiler, look at the test cases in this project and talk to me. I may have some ideas that I haven't made public yet. See also what I've wrtten about the on how this code works and on [decompilation in dynamic runtime languages](http://rocky.github.io/Deparsing-Paper.pdf) in general.
+That said, if you still feel you want to write a single version decompiler, look at the test cases in this project and talk to me. I may have some ideas that I haven't made public yet. See also what I've written about the on how this code works and on [decompilation in dynamic runtime languages](http://rocky.github.io/Deparsing-Paper.pdf) in general.
 
 
 
@@ -82,8 +82,8 @@ That said, if you still feel you want to write a single version decompiler, look
 
 This project deparses using an Earley-algorithm parse. But in order to do this accurately, the process of tokenization is a bit more involved in the scanner. We don't just disassemble bytecode and use the opcode name. That aspect hasn't changed from the very first decompilers. However understanding _what_ information needs to be made explicit and what pseudo instructions to add that accomplish this has taken some time to understand.
 
-Earley-algorithm parsers have gotten negative press, most notably by the dragon book. Having used this a bit, I am convinced having a system that handles ambiguous grammars is the right thing to do and matches the problem well. Iin practice the speed of the parser isn't a problem when one understand what's up. And this has taken a little while to understand.
-Earley-algorim parsers for context free languages or languages that are to a large extent context free and tend to be linear and the grammar stears towards left recursive rules. There is a technique for improving LL right recursion, but our parser doesn't have that yet.
+Earley-algorithm parsers have gotten negative press, most notably by the dragon book. Having used this a bit, I am convinced having a system that handles ambiguous grammars is the right thing to do and matches the problem well. In practice the speed of the parser isn't a problem when one understand what's up. And this has taken a little while to understand.
+Earley-algorithm parsers for context free languages or languages that are to a large extent context free and tend to be linear and the grammar steers towards left recursive rules. There is a technique for improving LL right recursion, but our parser doesn't have that yet.
 
 The [decompiling paper](http://rocky.github.io/Deparsing-Paper.pdf) discusses these aspects in a more detail.
 

HOW-TO-REPORT-A-BUG.md

@@ -22,7 +22,7 @@ TL;DR (too long; didn't read)
 * Don't do something illegal. And don't ask me to do something illegal or help you do something illegal
 * We already have an infinite supply of decompilation bugs that need fixing, and an automated mechanism for finding more. Decompilation bugs get addressed by easiness to fix and by whim. If you expect yours to be fixed ahead of those, you need to justify why.
 * When asking for help, you may be asked for what you've tried on your own first. There are plenty of sources of information about this code.
-* If you are looking for *timely* help or support, well, that is typically known paid service. I don't really have a mechanism for that since I have a full-time job. But supporting the project is an approximation.
+* If you are looking for *timely* help or support, well, that is typically known as a _paid_ service. I don't really have a mechanism for that since I have a full-time job. But supporting the project is an approximation.
 * Submitting a bug or issue report that is likely to get acted upon may require a bit of effort on your part to make it easy for the problem solver. If you are not willing to do that, please don't waste our time. As indicated above, supporting the project will increase the likelihood of your issue getting noticed and acted upon.
 
 # Ethics
@@ -37,7 +37,7 @@ confidentiality. You may be asked about the authorship or claimed ownership of t
 
 For many open-source projects bugs where the expectation is that bugs are rare, reporting bugs in a *thoughtful* way can be helpful. See also [How to Ask Questions the Smart Way](http://www.catb.org/~esr/faqs/smart-questions.html).
 
-In this project though, most of the bug reports boil down to the something like: I have I am trying to reverse engineer some code that I am not the author/owner and that person doesn't want me to have access to. I am hitting a problem somewhere along the line which might have to do with decompilation, but it could be something else like how the bytecode was extracted, some problem in deliberately obfuscated code, or the use some kind of Python bytecode version that isn't supported by the decompiler.
+In this project though, most of the bug reports boil down to the something like: I am trying to reverse engineer some code that I am not the author/owner and that person doesn't want me to have access to. I am hitting a problem somewhere along the line which might have to do with decompilation, but it could be something else like how the bytecode was extracted, some problem in deliberately obfuscated code, or the use some kind of Python bytecode version that isn't supported by the decompiler.
 
 While you are free to report these, unless you sponsor the project, I may close them with about the same amount of effort spent that I think was used to open the report for them. And if you spent a considerable amount of time to create the bug report but didn't follow instructions given here and in the issue template, I am sorry in advance. Just go back, read, and follow instructions.
 
@@ -74,7 +74,7 @@ obfuscation.
 Checking if bytecode is valid is pretty simple: disassemble the code.
 Python comes with a disassembly module called `dis`. A prerequisite
 module for this package, `xdis` has a cross-python version
-disassembler called `pydisasm`.
+disassembler called `pydisasm`. Using that with the `-F extended` option, generally provides a more comprehensive disassembly than is provided by other disassemblers.
 
 ## Semantic equivalence vs. exact source code
 

NEWS.md

@@ -1,3 +1,23 @@
+3.9.1: 2024-05-15
+=================
+
+Lots of changes major changes. track xdis API has changes.
+
+Separate Phases more clearly:
+* disassembly
+* tokenization
+* parsing
+* abstracting to AST (more is done in newer projects)
+* printing
+
+Although we do not decompile bytecode greater than 3.8, code supports running from up to 3.12.
+
+Many bugs fixed.
+
+A lot of Linting and coding style modernization.
+
+Work done in preparation for Blackhat Asia 2024
+
 3.9.0: 2022-12-22
 =================
 
@@ -8,7 +28,7 @@
 * Correct 2.5-7 relative import formatting
 * Miscellaneous bug fixing
 * remove \n in lambda
-* Python 2.6 gramar cleanup
+* Python 2.6 grammar cleanup
 * Correct some Python 2.6 chain compare decompilation
 * Ensure no parenthesis subscript slices
 * Correct 2.x formatting "slice2" nonterminal
@@ -35,7 +55,7 @@
 ================
 
 * Fragment parsing was borked. This means deparsing in trepan2/trepan3k was broken
-* 3.7+: narrow precedence for call tatement
+* 3.7+: narrow precedence for call statement
 * del_stmt -> delete to better match Python AST
 * 3.8+ Add another `forelsestmt` (found only in a loop)
 * 3.8+ Add precedence on walrus operator
@@ -66,7 +86,7 @@ Mostly small miscellaneous bug fixes
 3.7.1: 2020-6-12 Fleetwood66
 ====================================================
 
-Released to pick up new xdis version which has fixes to read bytestings better on 3.x
+Released to pick up new xdis version which has fixes to read bytestrings better on 3.x
 
 * Handle 3.7+ "else" branch removal adAs seen in `_cmp()` of `python3.8/distutils/version.py` with optimization `-O2`
 * 3.6+ "with" and "with .. as" grammar improvements
@@ -89,10 +109,10 @@ More upheaval in xdis which we need to track here.
 3.6.6: 2020-4-20 Love in the time of Cholera
 ============================================
 
-The main reason for this release is an incompatablity bump in xdis which handles
+The main reason for this release is an incompatibility bump in xdis which handles
 3.7 SipHash better.
 
-* Go over "yield" as an expression precidence
+* Go over "yield" as an expression precedence
 * Some small alignment with code in decompyle3 for "or" and "and" was done
 
 
@@ -118,7 +138,7 @@ The main focus in this release was fix some of the more glaring problems creapt
 `uncompyle6` code is at a plateau where what is most needed is a code refactoring. In doing this, until everything refactored and replaced, decomplation may get worse.
 Therefore, this release largely serves as a checkpoint before more major upheaval.
 
-The upheaval, in  started last release, I believe the pinnicle was around c90ff51 which wasn't a release. I suppose I should tag that.
+The upheaval, in  started last release, I believe the pinnacle was around c90ff51 which wasn't a release. I suppose I should tag that.
 
 After c90ff5, I started down the road of redoing control flow in a more comprehensible, debuggable, and scalable way. See [The Control Flow Mess](https://github.com/rocky/python-uncompyle6/wiki/The-Control-Flow-Mess)
 
@@ -132,7 +152,7 @@ In the decompyle3 code, I've gone down the road making the grammar goal symbol b
 
 I cringe in thinking about how the code has lived for so long without noticing such a simple stupidity, and lapse of sufficient thought.
 
-Some stats from testing. The below give numbers of decompiled tests from Python's test suite which succesfully ran
+Some stats from testing. The below give numbers of decompiled tests from Python's test suite which successfully ran
 
 ```
    Version  test-suites passing
@@ -175,14 +195,14 @@ On the most recent Python versions I regularly decompile thousands of Python pro
 
 Does this mean the decompiler works perfectly? No. There are still a dozen or so failing programs, although the actual number of bugs is probably smaller though.
 
-However, in perparation of a more major refactoring of the parser grammar, this release was born.
+However, in preparation of a more major refactoring of the parser grammar, this release was born.
 
 In many cases, decompilation is better. But there are some cases where decompilation has gotten worse. For lack of time (and interest) 3.0 bytecode suffered a hit. Possibly some code in the 3.x range did too. In time and with cleaner refactored code, this will come back.
 
-Commit c90ff51 was a local maxiumum before, I started reworking the grammar to separate productions that were specific to loops versus those that are not in loops.
-In the middle of that I added another grammar simplication to remove singleton productions of the form `sstmts-> stmts`. These were always was a bit ugly, and complicated output.
+Commit c90ff51 was a local maximum before, I started reworking the grammar to separate productions that were specific to loops versus those that are not in loops.
+In the middle of that I added another grammar simplification to remove singleton productions of the form `sstmts-> stmts`. These were always was a bit ugly, and complicated output.
 
-At any rate if decompilation fails, you can try c90ff51. Or another decompiler. `unpyc37` is pretty good for 3.7. wibiti `uncompyle2` is great for 2.7. `pycdc` is mediocre for Python before 3.5 or so, and not that good for the most recent Python. Geerally these programs will give some sort of answer even if it isn't correct.
+At any rate if decompilation fails, you can try c90ff51. Or another decompiler. `unpyc37` is pretty good for 3.7. wibiti `uncompyle2` is great for 2.7. `pycdc` is mediocre for Python before 3.5 or so, and not that good for the most recent Python. Generally these programs will give some sort of answer even if it isn't correct.
 
 decompyle3 isn't that good for 3.7 and worse for 3.8, but right now it does things no other Python decompiler like `unpyc37` or `pycdc` does. For example, `decompyle3` handles variable annotations. As always, the issue trackers for the various programs will give you a sense for what needs to be done. For now, I've given up on reporting issues in the other decompilers because there are already enough issues reported, and they are just not getting fixed anyway.
 
@@ -213,7 +233,7 @@ indicate when an import contains a dotted import. Similarly, code for
 3.7 `import .. as ` is basically the same as `from .. import`, the
 only difference is the target of the name changes to an "alias" in the
 former. As a result, the disambiguation is now done on the semantic
-action side, rathero than in parsing grammar rules.
+action side, rather than in parsing grammar rules.
 
 Some small specific fixes:
 
@@ -246,13 +266,13 @@ versions better. This however comes with a big decompilation speed
 penalty. When we redo control flow this should go back to normal, but
 for now, accuracy is more important than speed.
 
-Another `assert` transform rule was added. Parser rules to distingish
+Another `assert` transform rule was added. Parser rules to distinguish
 `try/finally` in 3.8 were added and we are more stringent about what
 can be turned into an `assert`. There was some grammar cleanup here
 too.
 
 A number of small bugs were fixed, and some administrative changes to
-make `make check-short` really be short, but check more throughly what
+make `make check-short` really be short, but check more thoroughly what
 it checks. minimum xdis version needed was bumped to include in the
 newer 3.6-3.9 releases. See the `ChangeLog` for details.
 
@@ -261,7 +281,7 @@ newer 3.6-3.9 releases. See the `ChangeLog` for details.
 =============================
 
 The main focus in this release was more accurate decompilation especially
-for 3.7 and 3.8. However there are some improvments to Python 2.x as well,
+for 3.7 and 3.8. However there are some improvements to Python 2.x as well,
 including one of the long-standing problems of detecting the difference between
 `try ... ` and `try else ...`.
 
@@ -269,11 +289,11 @@ With this release we now rebase Python 3.7 on off of a 3.7 base; This
 is also as it is (now) in decompyle3.  This facilitates removing some of the
 cruft in control-flow detection in the 2.7 uncompyle2 base.
 
-Alas, decompilation speed for 3.7 on is greatly increased. Hopefull
+Alas, decompilation speed for 3.7 on is greatly increased. Hopefully
 this is temporary (cough, cough) until we can do a static control flow
 pass.
 
-Finally, runing in 3.9-dev is tolerated. We can disassemble, but no parse tables yet.
+Finally, running in 3.9-dev is tolerated. We can disassemble, but no parse tables yet.
 
 
 3.5.1 2019-11-17 JNC
@@ -566,7 +586,7 @@ function calls and definitions.
 - Misc pydisasm fixes
 - Weird comprehension bug seen via new loctraceback
 - Fix Python 3.5+ CALL_FUNCTION_VAR and BUILD_LIST_UNPACK in call; with this
-  we can can handle 3.5+ f(a, b, *c, *d, *e) now
+  we can handle 3.5+ f(a, b, *c, *d, *e) now
 
 2.15.1 2018-02-05
 =====================
@@ -661,7 +681,7 @@ Overall: better 3.6 decompiling and some much needed code refactoring and cleanu
 - Handle `EXTENDED_ARGS` better. While relevant to all Python versions it is most noticeable in
   version 3.6+ where in switching to wordcodes the size of operands has been reduced from 2^16
   to 2^8. `JUMP` instruction then often need EXTENDED_ARGS.
-- Refactor find_jump_targets() with via working of of instructions rather the bytecode array.
+- Refactor find_jump_targets() with via working of instructions rather the bytecode array.
 - use `--weak-verify` more and additional fuzzing on verify()
 - fragment parser now ignores errors in nested function definitions; an parameter was
   added to assist here. Ignoring errors may be okay because the fragment parser often just needs,

NEW_FEATURES.rst

@@ -171,7 +171,7 @@ Expanding decompiler availability to multiple Python Versions
 --------------------------------------------------------------
 
 Above we mention decompiling multiple versions of bytecode from a
-single Python interpreter. We we talk about having the decompiler
+single Python interpreter. We talk about having the decompiler
 runnable from multiple versions of Python, independent of the set of
 bytecode that the decompiler supports.
 
@@ -185,7 +185,7 @@ implemented correctly. These also make excellent programs to check
 whether a program has decompiled correctly.
 
 Aside from this, debugging can be easier as well. To assist
-understanding bytcode and single stepping it see `x-python
+understanding bytecode and single stepping it see `x-python
 <https://pypi.org/project/x-python/>`_ and the debugger for it
 `trepan-xpy <https://pypi.org/project/trepanxpy/>`_.
 

README.rst

@@ -41,7 +41,7 @@ although compatible with the original intention, is yet a little bit
 different.  See this_ for more information.
 
 Python fragment deparsing given an instruction offset is useful in
-showing stack traces and can be encorporated into any program that
+showing stack traces and can be incorporated into any program that
 wants to show a location in more detail than just a line number at
 runtime.  This code can be also used when source-code information does
 not exist and there is just bytecode. Again, my debuggers make use of
@@ -161,8 +161,8 @@ Python syntax changes, you should use this option if the bytecode is
 the right bytecode for the Python interpreter that will be checking
 the syntax.
 
-You can also cross compare the results with either another version of
-`uncompyle6` since there are are sometimes regressions in decompiling
+You can also cross compare the results with another version of
+`uncompyle6` since there are sometimes regressions in decompiling
 specific bytecode as the overall quality improves.
 
 For Python 3.7 and 3.8, the code in decompyle3_ is generally
@@ -199,7 +199,7 @@ On the lower end of Python versions, decompilation seems pretty good although
 we don't have any automated testing in place for Python's distributed tests.
 Also, we don't have a Python interpreter for versions 1.6, and 2.0.
 
-In the Python 3 series, Python support is is strongest around 3.4 or
+In the Python 3 series, Python support is strongest around 3.4 or
 3.3 and drops off as you move further away from those versions. Python
 3.0 is weird in that it in some ways resembles 2.6 more than it does
 3.1 or 2.7. Python 3.6 changes things drastically by using word codes

__pkginfo__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018, 2020-2021 Rocky Bernstein <rocky@gnu.org>
+# Copyright (C) 2018, 2020-2021 2024 Rocky Bernstein <rocky@gnu.org>
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -32,9 +32,11 @@
 # 3.3            | pip     | 10.0.1       |
 # 3.4            | pip     | 19.1.1       |
 
+import os.path as osp
+
 # Things that change more often go here.
 copyright = """
-Copyright (C) 2015-2021 Rocky Bernstein <rb@dustyfeet.com>.
+Copyright (C) 2015-2021, 2024 Rocky Bernstein <rb@dustyfeet.com>.
 """
 
 classifiers = [
@@ -60,6 +62,8 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Software Development :: Debuggers",
     "Topic :: Software Development :: Libraries :: Python Modules",
@@ -75,7 +79,7 @@ entry_points = {
     ]
 }
 ftp_url = None
-install_requires = ["spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.0.2, < 6.2.0"]
+install_requires = ["click", "spark-parser >= 1.8.9, < 1.9.0", "xdis >= 6.0.8, < 6.2.0"]
 
 license = "GPL3"
 mailing_list = "python-debugger@googlegroups.com"
@@ -88,21 +92,18 @@ web = "https://github.com/rocky/python-uncompyle6/"
 zip_safe = True
 
 
-import os.path
-
-
 def get_srcdir():
-    filename = os.path.normcase(os.path.dirname(os.path.abspath(__file__)))
-    return os.path.realpath(filename)
+    filename = osp.normcase(osp.dirname(osp.abspath(__file__)))
+    return osp.realpath(filename)
 
 
 srcdir = get_srcdir()
 
 
 def read(*rnames):
-    return open(os.path.join(srcdir, *rnames)).read()
+    return open(osp.join(srcdir, *rnames)).read()
 
 
-# Get info from files; set: long_description and __version__
+# Get info from files; set: long_description and VERSION
 long_description = read("README.rst") + "\n"
 exec(read("uncompyle6/version.py"))

admin-tools/check-2.4-2.7-versions.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# Run tests over all Python versions in branch python-2.4-2.7
+set -e
 function finish {
   cd $owd
 }
@@ -10,7 +12,9 @@ if ! source ./pyenv-2.4-2.7-versions ; then
     exit $?
 fi
 if ! source ./setup-python-2.4.sh ; then
-    exit $?
+    rc=$?
+    finish
+    exit $rc
 fi
 
 cd ..
@@ -21,7 +25,10 @@ for version in $PYVERSIONS; do
     fi
     make clean && python setup.py develop
     if ! make check ; then
+	finish
+	rc=$?
 	exit $?
     fi
     echo === $version ===
 done
+finish

admin-tools/check-3.0-3.2-versions.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Run tests over all Python versions in branch python-3.0-3.2
+set -e
+function finish {
+  cd $owd
+}
+
+owd=$(pwd)
+trap finish EXIT
+
+cd $(dirname ${BASH_SOURCE[0]})
+if ! source ./pyenv-3.0-3.2-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.0.sh ; then
+    exit $?
+fi
+
+cd ..
+for version in $PYVERSIONS; do
+    echo --- $version ---
+    if ! pyenv local $version ; then
+	exit $?
+    fi
+    make clean && python setup.py develop
+    if ! make check ; then
+	exit $?
+    fi
+    echo === $version ===
+done
+finish

admin-tools/check-3.3-3.5-versions.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+# Run tests over all Python versions in branch python-3.3-3.5
+set -e
 function finish {
   cd $owd
 }
@@ -25,3 +27,4 @@ for version in $PYVERSIONS; do
     fi
     echo === $version ===
 done
+finish

admin-tools/make-dist-2.4-2.7.sh

@@ -3,9 +3,9 @@ PACKAGE=uncompyle6
 
 # FIXME put some of the below in a common routine
 function finish {
-  cd $owd
+  cd $make_dist_uncompyle6_owd
 }
-owd=$(pwd)
+make_dist_uncompyle6_owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
@@ -21,6 +21,11 @@ source $PACKAGE/version.py
 echo $__version__
 
 for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
@@ -41,3 +46,4 @@ tarball=dist/${PACKAGE}-${__version_}_-tar.gz
 if [[ -f $tarball ]]; then
     rm -v dist/${PACKAGE}-${__version__}-tar.gz
 fi
+finish

admin-tools/make-dist-3.0-3.2.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+PACKAGE=uncompyle6
+
+# FIXME put some of the below in a common routine
+function finish {
+  cd $uncompyle6_30_make_dist_owd
+}
+
+cd $(dirname ${BASH_SOURCE[0]})
+uncompyle6_30_make_dist_owd=$(pwd)
+trap finish EXIT
+
+if ! source ./pyenv-3.0-3.2-versions ; then
+    exit $?
+fi
+if ! source ./setup-python-3.0.sh ; then
+    exit $?
+fi
+
+cd ..
+source $PACKAGE/version.py
+echo $__version__
+
+for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
+    if ! pyenv local $pyversion ; then
+	exit $?
+    fi
+    # pip bdist_egg create too-general wheels. So
+    # we narrow that by moving the generated wheel.
+
+    # Pick out first two number of version, e.g. 3.5.1 -> 35
+    first_two=$(echo $pyversion | cut -d'.' -f 1-2 | sed -e 's/\.//')
+    rm -fr build
+    python setup.py bdist_egg bdist_wheel
+    mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
+    echo === $pyversion ===
+done
+
+python ./setup.py sdist
+tarball=dist/${PACKAGE}-${__version__}.tar.gz
+if [[ -f $tarball ]]; then
+    mv -v $tarball dist/${PACKAGE}_31-${__version__}.tar.gz
+fi
+finish

admin-tools/make-dist-3.3-3.5.sh

@@ -3,11 +3,11 @@ PACKAGE=uncompyle6
 
 # FIXME put some of the below in a common routine
 function finish {
-  cd $owd
+  cd $uncompyle6_33_make_owd
 }
 
 cd $(dirname ${BASH_SOURCE[0]})
-owd=$(pwd)
+uncompyle6_33_make_owd=$(pwd)
 trap finish EXIT
 
 if ! source ./pyenv-3.3-3.5-versions ; then
@@ -22,6 +22,11 @@ source $PACKAGE/version.py
 echo $__version__
 
 for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
@@ -33,6 +38,12 @@ for pyversion in $PYVERSIONS; do
     rm -fr build
     python setup.py bdist_egg bdist_wheel
     mv -v dist/${PACKAGE}-$__version__-{py2.py3,py$first_two}-none-any.whl
+    echo === $pyversion ===
 done
 
 python ./setup.py sdist
+tarball=dist/${PACKAGE}-${__version__}.tar.gz
+if [[ -f $tarball ]]; then
+    mv -v $tarball dist/${PACKAGE}_31-${__version__}.tar.gz
+fi
+finish

admin-tools/make-dist-newest.sh

@@ -3,11 +3,11 @@ PACKAGE=uncompyle6
 
 # FIXME put some of the below in a common routine
 function finish {
-  cd $owd
+  cd $make_uncompyle6_newest_owd
 }
 
 cd $(dirname ${BASH_SOURCE[0]})
-owd=$(pwd)
+make_uncompyle6_newest_owd=$(pwd)
 trap finish EXIT
 
 if ! source ./pyenv-newest-versions ; then
@@ -22,6 +22,11 @@ source $PACKAGE/version.py
 echo $__version__
 
 for pyversion in $PYVERSIONS; do
+    echo --- $pyversion ---
+    if [[ ${pyversion:0:4} == "pypy" ]] ; then
+	echo "$pyversion - PyPy does not get special packaging"
+	continue
+    fi
     if ! pyenv local $pyversion ; then
 	exit $?
     fi
@@ -36,3 +41,4 @@ for pyversion in $PYVERSIONS; do
 done
 
 python ./setup.py sdist
+finish

admin-tools/merge-for-2.4.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_24_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-2.4.sh; then
+    git merge python-3.0-to-3.2
+fi
+cd $uncompyle6_merge_24_owd

admin-tools/merge-for-3.0.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_30_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-3.0.sh; then
+    git merge python-3.3-to-3.5
+fi
+cd $uncompyle6_merge_30_owd

admin-tools/merge-for-3.3.sh

@@ -0,0 +1,7 @@
+#/bin/bash
+uncompyle6_merge_33_owd=$(pwd)
+cd $(dirname ${BASH_SOURCE[0]})
+if . ./setup-python-3.3.sh; then
+    git merge master
+fi
+cd $uncompyle6_merge_33_owd

admin-tools/pyenv-3.3-3.5-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.5.10 3.3.7 3.4.10'
+export PYVERSIONS=' 3.3.7 3.4.10 3.5.10 '

admin-tools/pyenv-newest-versions

@@ -5,4 +5,4 @@ if [[ $0 == ${BASH_SOURCE[0]} ]] ; then
     echo "This script should be *sourced* rather than run directly through bash"
     exit 1
 fi
-export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.16'
+export PYVERSIONS='3.6.15 pypy3.6-7.3.1 3.7.16 pypy3.7-7.3.9 pypy3.8-7.3.10 pyston-2.3.5 3.8.18'

admin-tools/setup-master.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
-PYTHON_VERSION=3.7.16
+# Check out master branch and dependent development master branches
+PYTHON_VERSION=3.8.18
+
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
 
 function checkout_version {
     local repo=$1
@@ -10,22 +17,16 @@ function checkout_version {
     return $?
 }
 
-# FIXME put some of the below in a common routine
-function finish {
-  cd $owd
-}
+owd=$(pwd)
 
 export PATH=$HOME/.pyenv/bin/pyenv:$PATH
-owd=$(pwd)
-bs=${BASH_SOURCE[0]}
-if [[ $0 == $bs ]] ; then
-    echo "This script should be *sourced* rather than run directly through bash"
-    exit 1
-fi
+
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
 (cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis &&
-      checkout_version python-uncompyle6)
+     checkout_version python-uncompyle6)
+
+git pull
+rm -v */.python-version || true
 cd $owd
-rm -v */.python-version >/dev/null 2>&1 || true

admin-tools/setup-python-2.4.sh

@@ -1,24 +1,32 @@
 #!/bin/bash
+# Check out python-2.4-to-2.7 and dependent development branches.
+
 PYTHON_VERSION=2.4.6
 
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
+
 function checkout_version {
     local repo=$1
-    version=${2:-python-2.4}
-    echo Checking out $version.4 on $repo ...
+    version=${2:-python-2.4-to-2.7}
+    echo Checking out $version on $repo ...
     (cd ../$repo && git checkout $version && pyenv local $PYTHON_VERSION) && \
 	git pull
     return $?
 }
 
 owd=$(pwd)
-bs=${BASH_SOURCE[0]}
-if [[ $0 == $bs ]] ; then
-    echo "This script should be *sourced* rather than run directly through bash"
-    exit 1
-fi
+
+export PATH=$HOME/.pyenv/bin/pyenv:$PATH
+
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 (cd $fulldir/.. && checkout_version python-spark && checkout_version python-xdis python-2.4-to-2.7 &&
-      checkout_version python-uncompyle6)
-cd $owd
+     checkout_version python-uncompyle6)
+
+git pull
 rm -v */.python-version || true
+cd $owd

admin-tools/setup-python-3.0.sh

@@ -1,6 +1,13 @@
 #!/bin/bash
+# Check out python-3.0-to-3.2 and dependent development branches.
+
 PYTHON_VERSION=3.0.1
-pyenv local $PYTHON_VERSION
+
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
 
 # FIXME put some of the below in a common routine
 function checkout_version {
@@ -12,24 +19,17 @@ function checkout_version {
     return $?
 }
 
-function finish {
-  cd $owd
-}
+owd=$(pwd)
+trap finish EXIT
 
 export PATH=$HOME/.pyenv/bin/pyenv:$PATH
-owd=$(pwd)
-bs=${BASH_SOURCE[0]}
-if [[ $0 == $bs ]] ; then
-    echo "This script should be *sourced* rather than run directly through bash"
-    exit 1
-fi
 
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
 (cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
       checkout_version python-uncompyle6)
-cd $owd
-rm -v */.python-version || true
 
-git checkout python-3.0-to-3.2  && git pull && pyenv local $PYTHON_VERSION
+git pull
+rm -v */.python-version || true
+cd $owd

admin-tools/setup-python-3.3.sh

@@ -1,6 +1,12 @@
 #!/bin/bash
+# Check out python-3.3-to-3.5 and dependent development branches.
 PYTHON_VERSION=3.3.7
-pyenv local $PYTHON_VERSION
+
+bs=${BASH_SOURCE[0]}
+if [[ $0 == $bs ]] ; then
+    echo "This script should be *sourced* rather than run directly through bash"
+    exit 1
+fi
 
 # FIXME put some of the below in a common routine
 function checkout_version {
@@ -12,24 +18,17 @@ function checkout_version {
     return $?
 }
 
-function finish {
-  cd $owd
-}
+owd=$(pwd)
 
 export PATH=$HOME/.pyenv/bin/pyenv:$PATH
-owd=$(pwd)
-bs=${BASH_SOURCE[0]}
-if [[ $0 == $bs ]] ; then
-    echo "This script should be *sourced* rather than run directly through bash"
-    exit 1
-fi
 
 mydir=$(dirname $bs)
 fulldir=$(readlink -f $mydir)
 cd $fulldir/..
 (cd $fulldir/.. && checkout_version python-spark master && checkout_version python-xdis &&
-      checkout_version python-uncompyle6)
-cd $owd
+     checkout_version python-uncompyle6)
 rm -v */.python-version || true
 
-git checkout python-3.3-to-3.5  && git pull && pyenv local $PYTHON_VERSION
+git pull
+rm -v */.python-version || true
+cd $owd

pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = [
+    "setuptools>=61.2",
+]
+
+build-backend = "setuptools.build_meta"
+
+[project]
+authors = [
+    {name = "Rocky Bernstein",	email = "rb@dustyfeet.com"},
+]
+
+name = "uncompyle6"
+description = "Python cross-version byte-code library and disassembler"
+dependencies = [
+    "click",
+    "spark-parser >= 1.8.9, < 1.9.0",
+    "xdis >= 6.0.8, < 6.2.0",
+]
+readme = "README.rst"
+license = {text = "GPL"}
+keywords = ["Python bytecode", "bytecode", "disassembler"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Programming Language :: Python :: 2.4",
+    "Programming Language :: Python :: 2.5",
+    "Programming Language :: Python :: 2.6",
+    "Programming Language :: Python :: 2.7",
+    "Programming Language :: Python :: 3.0",
+    "Programming Language :: Python :: 3.1",
+    "Programming Language :: Python :: 3.2",
+    "Programming Language :: Python :: 3.3",
+    "Programming Language :: Python :: 3.4",
+    "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/rocky/python-uncompyle6"
+Downloads = "https://github.com/rocky/python-uncompyle6/releases"
+
+[project.optional-dependencies]
+dev = [
+    "pre-commit",
+    "pytest",
+]
+
+[project.scripts]
+uncompyle6 = "uncompyle6.bin.uncompile:main_bin"
+uncompyle6-tokenize = "uncompyle6.bin.pydisassemble:main"
+
+[tool.setuptools.dynamic]
+version = {attr = "uncompyle6.version.__version__"}

pytest/test_fjt.py

@@ -27,7 +27,7 @@ def test_if_in_for():
         fjt = scan.find_jump_targets(False)
 
         ## FIXME: the data below is wrong.
-        ## we get different results currenty as well.
+        ## we get different results currently as well.
         ## We need to probably fix both the code
         ## and the test below
         # assert {15: [3], 69: [66], 63: [18]} == fjt

pytest/test_grammar.py

@@ -123,6 +123,7 @@ def test_grammar():
             opcode_set.add("THEN")
         check_tokens(tokens, opcode_set)
     elif PYTHON_VERSION_TRIPLE[:2] == (3, 4):
+        ignore_set.add("LOAD_ARG")  # Used in grammar for comprehension. But not in 3.4
         ignore_set.add("LOAD_CLASSNAME")
         ignore_set.add("STORE_LOCALS")
         opcode_set = set(s.opc.opname).union(ignore_set)

pytest/validate.py

@@ -67,7 +67,7 @@ def are_instructions_equal(i1, i2):
     Determine if two instructions are approximately equal,
     ignoring certain fields which we allow to differ, namely:
 
-    * code objects are ignore (should probaby be checked) due to address
+    * code objects are ignore (should probably be checked) due to address
     * line numbers
 
     :param i1: left instruction to compare

setup.py

@@ -5,16 +5,21 @@ import sys
 """Setup script for the 'uncompyle6' distribution."""
 
 SYS_VERSION = sys.version_info[0:2]
-if not ((2, 4) <= SYS_VERSION < (3, 12)):
-    mess = "Python Release 2.6 .. 3.11 are supported in this code branch."
+if not ((3, 0) <= SYS_VERSION < (3, 3)):
+    mess = "Python Release 3.0 .. 3.2 are supported in this code branch."
     if (2, 4) <= SYS_VERSION <= (2, 7):
         mess += (
             "\nFor your Python, version %s, use the python-2.4 code/branch."
             % sys.version[0:3]
         )
-    if (3, 3) <= SYS_VERSION < (3, 6):
+    if SYS_VERSION >= (3, 6):
         mess += (
-            "\nFor your Python, version %s, use the python-3.3-to-3.5 code/branch."
+            "\nFor your Python, version %s, use the master code/branch."
+            % sys.version[0:3]
+        )
+    if (3, 3) >= SYS_VERSION < (3, 6):
+        mess += (
+            "\nFor your Python, version %s, use the python-3.3-to-3.6 code/branch."
             % sys.version[0:3]
         )
     elif SYS_VERSION < (2, 4):

test/Makefile

@@ -115,7 +115,7 @@ check-bytecode-2:
 # FIXME: Until we shaked out problems with xdis...
 check-bytecode-3:
 	$(PYTHON) test_pythonlib.py \
-	--bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
+	--bytecode-3.3 --bytecode-3.4 --bytecode-3.5 --bytecode-3.6 \
         --bytecode-3.7  --bytecode-3.8
 
 #: Check deparsing on selected bytecode 3.x

test/decompyle/test_prettyprint.py

@@ -1,6 +1,6 @@
 """
 test_prettyprint.py --	source test pattern for tesing the prettyprint
-			funcionality of decompyle
+			functionality of decompyle
 
 This source is part of the decompyle test suite.
 

test/simple_source/bug27+/03_comprehension_in_lambda.py

@@ -0,0 +1,11 @@
+# RUNNABLE!
+# From issue 469
+
+"""This program is self-checking!"""
+
+my_dict = (lambda variable0: {variable1: 123 for variable1 in variable0})([1, 2, 3])
+
+assert my_dict[1] == 123
+
+my_set = (lambda variable0: {variable1 for variable1 in variable0})([1, 2, 3])
+assert 2 in my_set

test/simple_source/bug27+/04_try_tryelse.py

@@ -1,5 +1,5 @@
 # From 2.7 test_normalize.py
-# Bug has to to with finding the end of the tryelse block. I think thrown
+# Bug has to do with finding the end of the tryelse block. I think thrown
 # off by the "continue". In instructions the COME_FROM for END_FINALLY
 # was at the wrong offset because some sort of "rtarget" was adjust.
 

test/simple_source/bug33/01_triple_compare.py

@@ -1,7 +1,7 @@
 # In Python 3.3+ this uses grammar rule
-# compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
+# compare_chained_right ::= expr COMPARE_OP RETURN_VALUE
 # In Python 3.6 uses this uses grammar rule
-# compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
+# compare_chained_right ::= expr COMPARE_OP come_froms JUMP_FORWARD
 
 # Seen in Python 3.3 ipaddress.py
 

test/simple_source/bug36/02_kwargs.py

@@ -5,7 +5,7 @@ def bug(self, j, a, b):
     self.parse_comment(a, b, report=3)
 
 # From 3.6 fnmatch.py
-# Bug was precidence parenthesis around decorator
+# Bug was precedence parenthesis around decorator
 
 import functools
 @functools.lru_cache(maxsize=256, typed=True)

test/simple_source/expression/05_long_literals.py

@@ -725,3 +725,13 @@ values = {
 }
 
 assert sorted(values.values())[1:] == list(range(2, 34))
+
+def assert_equal(x, y):
+    assert x == y
+
+# Check that we can distinguish names from strings in literal collections, e.g. lists.
+# The list has to have more than 4 items to get accumulated in a collection
+a = ["y", 'Exception', "x", Exception, "z"]
+
+assert_equal(a[1], "Exception")
+assert_equal(a[3], Exception)

test/simple_source/stmts/08_test_contextmanager.py

@@ -0,0 +1,21 @@
+"""
+This program is self checking!
+"""
+
+
+class TestContextManager:
+    def __enter__(self):
+        return 1, 2
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        return self, exc_type, exc_value, exc_tb
+
+
+with open(__file__) as a:
+    assert a
+
+with open(__file__) as a, open(__file__) as b:
+    assert a.read() == b.read()
+
+with TestContextManager() as a, b:
+    assert (a, b) == (1, 2)

test/simple_source/stmts/11_return_val.py

@@ -1,6 +1,6 @@
 # 2.5.6 decimal.py
 # Bug on 2.5 and 2.6 by incorrectly changing opcode to
-# RETURN_VALUE to psuedo op: RETURN_END_IF
+# RETURN_VALUE to pseudo op: RETURN_END_IF
 def _formatparam(param, value=None, quote=True):
     if value is not None and len(value) > 0:
         if isinstance(value, tuple):

test/stdlib/2.5-exclude.sh

@@ -25,7 +25,6 @@ SKIP_TESTS=(
     [test_nis.py]=1 # it fails on its own
     [test_normalization.py]=1 # it fails on its own
     [test_ossaudiodev.py]=1 # it fails on its own
-    [test_pep277.py]=1 # it fails on its own
     [test_plistlib.py]=1 # it fails on its own
     [test_rgbimg.py]=1 # it fails on its own
     [test_scriptpackages.py]=1 # it fails on its own

test/stdlib/2.6-exclude.sh

@@ -8,23 +8,23 @@ SKIP_TESTS=(
     # [test_shutil.py]=1 # OK but needs PYTHON=pytest
 
     [test___all__.py]=1 # it fails on its own
-    [test_aepack.py]=1 # it fails on its own
-    [test_al.py]=1 # it fails on its own
-    [test_anydbm.py]=1 # it fails on its own
-    [test_applesingle.py]=1 # it fails on its own
-
+    [test_aepack.py]=1 # No module macostools
+    [test_al.py]=1 # No module macostools
+    [test_anydbm.py]=pytest
+    [test_applesingle.py]=1 # No module macostools
     [test_bsddb185.py]=1 # it fails on its own
     [test_bsddb3.py]=1 # it fails on its own
-    [test_bsddb.py]=1 # it fails on its own
+    [test_bsddb.py]=1 # No module _bsdb
 
-    # [test_cd.py]=1 # it fails on its own
+    [test_cd.py]=1 # i# No module cl
     [test_cl.py]=1 # it fails on its own
+    [test_cmath.py]=pytest
     [test_codecmaps_cn.py]=1 # it fails on its own
     [test_codecmaps_jp.py]=1 # it fails on its own
     [test_codecmaps_kr.py]=1 # it fails on its own
     [test_codecmaps_tw.py]=1 # it fails on its own
     [test_commands.py]=1 # it fails on its own
-    [test_curses.py]=1 # it fails on its own
+    [test_curses.py]=1 # needs libncurses.so.5
 
     [test_dbm.py]=1 # it fails on its own
     [test_descr.py]=1
@@ -33,16 +33,16 @@ SKIP_TESTS=(
     [test_dl.py]=1 # it fails on its own
 
     [test_file.py]=1 # it fails on its own
-    [test_future5.py]=1 # it fails on its own
+    [test_future5.py]=pytest
 
-    # [test_generators.py]=1 # works but use PYTHON=pytest
+    [test_generators.py]=pytest
     [test_gl.py]=1 # it fails on its own
-    # [test_grp.py]=1  # works but use PYTHON=pytest
+    [test_grp.py]=pytest
 
     [test_imageop.py]=1 # it fails on its own
     [test_imaplib.py]=1 # it fails on its own
     [test_imgfile.py]=1 # it fails on its own
-    # [test_ioctl.py]=1  # works but use PYTHON=pytest
+    [test_ioctl.py]=pytest
 
     [test_kqueue.py]=1 # it fails on its own
 
@@ -63,6 +63,7 @@ SKIP_TESTS=(
     [test_scriptpackages.py]=1 # it fails on its own
     [test_select.py]=1 # test takes too long to run: 11 seconds
 
+    [test_signal.py]=1 # takes more than 15 seconds to run
     [test_socket.py]=1 # test takes too long to run: 12 seconds
     [test_startfile.py]=1 # it fails on its own
     [test_structmembers.py]=1 # it fails on its own
@@ -81,8 +82,8 @@ SKIP_TESTS=(
     [test_winreg.py]=1 # it fails on its own
     [test_winsound.py]=1 # it fails on its own
 
-    [test_zipimport_support.py]=1 # expected test to raise ImportError
-    [test_zipfile64.py]=1  # Skip Long test
+    [test_zipimport_support.py]=pytest # expected test to raise ImportError
+    [test_zipfile.py]=pytest  # Skip Long test
     # .pyenv/versions/2.6.9/lib/python2.6/lib2to3/refactor.pyc
     # .pyenv/versions/2.6.9/lib/python2.6/pyclbr.pyc
 )

test/stdlib/3.2-exclude.sh

@@ -1,15 +1,42 @@
 SKIP_TESTS=(
-    [test_descr.py]=1 # FIXME: Works on c90ff51?
+    [test_descr.py]=1
+    # [test_descr.py]=pytest_module # FIXME: Works on c90ff51?
+    # AssertionError: 'D(4)C(4)A(4)' != 'D(4)C(4)B(4)A(4)'
+    # - D(4)C(4)A(4)
+    # + D(4)C(4)B(4)A(4)
+    # ?         ++++
+
+
+    [test_cmath.py]=1 # Control-flow "elif else -> else: if else"
+    # [test_cmath.py]=pytest_module
+    # AssertionError: rect1000: rect(complex(0.0, 0.0))
+    # Expected: complex(0.0, 0.0)
+    # Received: complex(0.0, -1.0)
+    # Received value insufficiently close to expected value.
+
 
     [test_cmd_line.py]=1
-    [test_collections.py]=1
+
+    [test_collections.py]=1 # fail on its own
+    # E   TypeError: __new__() takes exactly 4 arguments (1 given)
+
     [test_concurrent_futures.py]=1 # too long to run over 46 seconds by itself
-    [test_datetimetester.py]=1
-    [test_decimal.py]=1
-    [test_dictcomps.py]=1 # FIXME: semantic error: actual = {k:v for k in }
-    [test_doctest.py]=1 # test failures
+    [test_datetime.py]=pytest_module
+
+    [test_decimal.py]=1 # Fails on its own, even with pytest
+
+    [test_dictcomps.py]=1
+    # [test_dictcomps.py]=pytest_module # FIXME: semantic error: actual = {k:v for k in }
+    #    assert (count * 2) <= i
+
+    [test_doctest.py]=1 # Missing pytest fixture
+    # [test_doctest.py]=pytest_module
+    # fixture 'coverdir' not found
+
     [test_dis.py]=1   # We change line numbers - duh!
+
     [test_exceptions.py]=1 # parse error
+    # [test_exceptions.py]=pytest_module # parse error
 
     [test_modulefinder.py]=1 # test failures
     [test_multiprocessing.py]=1 # test takes too long to run: 35 seconds

test/stdlib/3.3-exclude.sh

@@ -10,8 +10,8 @@ SKIP_TESTS=(
     #       tgt.append(elem)
     [test_itertools.py]=1
 
-    [test_buffer.py]=1  # FIXME: Works on c90ff51
-    [test_cmath.py]=1  # FIXME: Works on c90ff51
+    [test_buffer.py]=pytest
+    [test_cmath.py]=pytest
 
     [test_atexit.py]=1  # The atexit test starting at 3.3 looks for specific comments in error lines
 
@@ -19,7 +19,6 @@ SKIP_TESTS=(
     [test_concurrent_futures.py]=1  # too long?
 
     [test_decimal.py]=1 # test takes too long to run: 18 seconds
-    [test_descr.py]=1  # test assertion errors
     [test_doctest.py]=1  # test assertion errors
     [test_doctest2.py]=1  # test assertion errors
     [test_dis.py]=1   # We change line numbers - duh!

test/stdlib/3.4-exclude.sh

@@ -26,7 +26,16 @@ SKIP_TESTS=(
 
     [test_dbm_gnu.py]=1   # fails on its own
     [test_devpoll.py]=1 # it fails on its own
+
     [test_descr.py]=1   # test assertion errors
+    # ERROR: test_reent_set_bases_on_base (__main__.MroTest)
+    # Traceback (most recent call last):
+    # File "test_descr.py", line 5521, in test_reent_set_bases_on_base
+    # class A(metaclass=M):
+    # File "test_descr.py", line 5472, in __new__
+    # return type.__new__(mcls, name, bases, attrs)
+    # TypeError: 'NoneType' object is not iterable
+
     [test_dis.py]=1   # We change line numbers - duh!
     [test_distutils.py]=1 # it fails on its own
     [test_doctest2.py]=1

test/stdlib/3.6-exclude.sh

@@ -1,6 +1,6 @@
 SKIP_TESTS=(
     [test_ast.py]=1 # FIXME: Works on c90ff51
-    [test_cmath.py]=1  # FIXME: Works on c90ff51
+    [test_cmath.py]=1  # fails on its own
     [test_format.py]=1 # FIXME: Works on c90ff51
     [test_ftplib.py]=1 # FIXME: Works on c90ff51
     [test_slice.py]=1 # FIXME: Works on c90ff51

test/stdlib/3.7-exclude.sh

@@ -36,7 +36,7 @@ SKIP_TESTS=(
     [test_bdb.py]=1  #
     [test_buffer.py]=1  # parse error
     [test_clinic.py]=1 # it fails on its own
-    [test_cmath.py]=1 # test assertion failure
+    [test_cmath.py]=pytest
     [test_cmd_line.py]=1  # Interactive?
     [test_cmd_line_script.py]=1
     [test_compileall.py]=1 # fails on its own

test/stdlib/3.8-exclude.sh

@@ -28,12 +28,12 @@ SKIP_TESTS=(
 
     # These and the above may be due to new code generation or tests
     # between 3.8.3 and 3.8.5 ?
-    [test_decorators.py]=1 #
+    [test_decorators.py]=1 # parse error
 
-    [test_dtrace.py]=1 #
-    [test_exceptions.py]=1 #
+    [test_dtrace.py]=1 # parse error
+    [test_exceptions.py]=1 # parse error
     [test_ftplib.py]=1 #
-    [test_gc.py]=1 #
+    [test_gc.py]=1 # FIXME: return                 return strip_python_stderr(stderr)
     [test_gzip.py]=1 #
     [test_hashlib.py]=1 #
     [test_iter.py]=1 #
@@ -51,7 +51,6 @@ SKIP_TESTS=(
     [test_audioop.py]=1  # test failure
     [test_audit.py]=1  # parse error
 
-    [test_base64.py]=1  # parse error
     [test_baseexception.py]=1  #
     [test_bigaddrspace.py]=1  # parse error
     [test_bigmem.py]=1  # parse error
@@ -69,7 +68,7 @@ SKIP_TESTS=(
     [test_cgi.py]=1  # parse error
     [test_cgitb.py]=1  # parse error
     [test_clinic.py]=1 # it fails on its own
-    [test_cmath.py]=1 # test assertion failure
+    [test_cmath.py]=pytest
     [test_cmd.py]=1  # parse error
     [test_cmd_line.py]=1  # Interactive?
     [test_cmd_line_script.py]=1

test/stdlib/runtests.sh

@@ -168,12 +168,8 @@ if ((IS_PYPY)); then
 else
     cp -r ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test $TESTDIR
 fi
-if [[ $PYVERSION == 3.2 ]] ; then
-    cp ${PYENV_ROOT}/versions/${PYVERSION}.${MINOR}/lib/python${PYVERSION}/test/* $TESTDIR
-    cd $TESTDIR
-else
-    cd $TESTDIR/test
-fi
+cd $TESTDIR/test
+
 pyenv local $FULLVERSION
 export PYTHONPATH=$TESTDIR
 export PATH=${PYENV_ROOT}/shims:${PATH}
@@ -187,7 +183,11 @@ if [[ -n $1 ]] ; then
     files=$@
     typeset -a files_ary=( $(echo $@) )
     if (( ${#files_ary[@]} == 1 || DONT_SKIP_TESTS == 1 )) ; then
-	SKIP_TESTS=()
+	for file in $files; do
+	    if (( SKIP_TESTS[$file] != "pytest" || SKIP_TESTS[$file] != "pytest_module" )); then
+	       SKIP_TESTS[$file]=1;
+	    fi
+	done
     fi
 else
     files=$(echo test_*.py)
@@ -201,9 +201,16 @@ NOT_INVERTED_TESTS=${NOT_INVERTED_TESTS:-1}
 for file in $files; do
     # AIX bash doesn't grok [[ -v SKIP... ]]
     [[ -z ${SKIP_TESTS[$file]} ]] && SKIP_TESTS[$file]=0
-    if [[ ${SKIP_TESTS[$file]} == ${NOT_INVERTED_TESTS} ]] ; then
-	((skipped++))
-	continue
+
+    if [[ ${SKIP_TESTS[$file]} == "pytest" ]]; then
+	PYTHON=pytest
+    elif [[ ${SKIP_TESTS[$file]} == "pytest_module" ]]; then
+	PYTHON="$PYTHON -m pytest"
+    else
+	if [[ ${SKIP_TESTS[$file]}s == ${NOT_INVERTED_TESTS} ]] ; then
+	    ((skipped++))
+	    continue
+	fi
     fi
 
     # If the fails *before* decompiling, skip it!

test/test-xpython.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Checks xpython passes all runnable bytecode here
+
+for dir in bytecode_*_run; do
+    for file in ${dir}/*.pyc; do
+	echo $file
+	if ! xpython $file; then
+	    break
+	fi
+    done
+done

test/test_pythonlib.py

@@ -216,7 +216,7 @@ def do_tests(src_dir, obj_patterns, target_dir, opts):
     print("Output directory: ", target_dir)
     try:
         _, _, failed_files, failed_verify = main(
-            src_dir, target_dir, files, [], do_verify=opts["do_verify"]
+            src_dir, target_dir, files, []
         )
         if failed_files != 0:
             sys.exit(2)

uncompyle6/bin/pydisassemble.py

@@ -1,10 +1,25 @@
 #!/usr/bin/env python
-# Mode: -*- python -*-
 #
-# Copyright (c) 2015-2016, 2018, 2020, 2022 by Rocky Bernstein <rb@dustyfeet.com>
+#  Copyright (c) 2015-2016, 2018, 2020, 2022-2024
+#  by Rocky Bernstein <rb@dustyfeet.com>
 #
-from __future__ import print_function
-import sys, os, getopt
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import getopt
+import os
+import sys
 
 from uncompyle6.code_fns import disassemble_file
 from uncompyle6.version import __version__
@@ -20,7 +35,7 @@ Disassemble/Tokenize FILE with in the way that is done to
 assist uncompyle6 in parsing the instruction stream. For example
 instructions with variable-length arguments like CALL_FUNCTION and
 BUILD_LIST have argument counts appended to the instruction name, and
-COME_FROM psuedo instructions are inserted into the instruction stream.
+COME_FROM pseudo instructions are inserted into the instruction stream.
 Bit flag values encoded in an operand are expanding, EXTENDED_ARG
 value are folded into the following instruction operand.
 
@@ -40,47 +55,54 @@ Options:
   -V | --version     show version and stop
   -h | --help        show this message
 
-""".format(program)
+""".format(
+    program
+)
+
+PATTERNS = ("*.pyc", "*.pyo")
 
-PATTERNS = ('*.pyc', '*.pyo')
 
 def main():
-    Usage_short = """usage: %s FILE...
-Type -h for for full help.""" % program
+    usage_short = (
+        """usage: %s FILE...
+Type -h for for full help."""
+        % program
+    )
 
     if len(sys.argv) == 1:
         print("No file(s) given", file=sys.stderr)
-        print(Usage_short, file=sys.stderr)
+        print(usage_short, file=sys.stderr)
         sys.exit(1)
 
     try:
-        opts, files = getopt.getopt(sys.argv[1:], 'hVU',
-                                    ['help', 'version', 'uncompyle6'])
+        opts, files = getopt.getopt(
+            sys.argv[1:], "hVU", ["help", "version", "uncompyle6"]
+        )
     except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
+        print("%s: %s" % (os.path.basename(sys.argv[0]), e), file=sys.stderr)
         sys.exit(-1)
 
     for opt, val in opts:
-        if opt in ('-h', '--help'):
+        if opt in ("-h", "--help"):
             print(__doc__)
             sys.exit(1)
-        elif opt in ('-V', '--version'):
+        elif opt in ("-V", "--version"):
             print("%s %s" % (program, __version__))
             sys.exit(0)
         else:
             print(opt)
-            print(Usage_short, file=sys.stderr)
+            print(usage_short, file=sys.stderr)
             sys.exit(1)
 
     for file in files:
         if os.path.exists(files[0]):
             disassemble_file(file, sys.stdout)
         else:
-            print("Can't read %s - skipping" % files[0],
-                  file=sys.stderr)
+            print("Can't read %s - skipping" % files[0], file=sys.stderr)
             pass
         pass
     return
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main()

uncompyle6/bin/uncompile.py

@@ -1,170 +1,195 @@
 #!/usr/bin/env python
 # Mode: -*- python -*-
 #
-# Copyright (c) 2015-2017, 2019-2020 by Rocky Bernstein
+# Copyright (c) 2015-2017, 2019-2020, 2023-2024
+# by Rocky Bernstein
 # Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
-from __future__ import print_function
-import sys, os, getopt, time
+
+import os
+import sys
+import time
+
+import click
 from xdis.version_info import version_tuple_to_str
 
-program = 'uncompyle6'
-
-__doc__ = """
-Usage:
-  %s [OPTIONS]... [ FILE | DIR]...
-  %s [--help | -h | --V | --version]
-
-Examples:
-  %s      foo.pyc bar.pyc       # decompile foo.pyc, bar.pyc to stdout
-  %s -o . foo.pyc bar.pyc       # decompile to ./foo.pyc_dis and ./bar.pyc_dis
-  %s -o /tmp /usr/lib/python1.5 # decompile whole library
-
-Options:
-  -o <path>     output decompiled files to this path:
-                if multiple input files are decompiled, the common prefix
-                is stripped from these names and the remainder appended to
-                <path>
-                  uncompyle6 -o /tmp bla/fasel.pyc bla/foo.pyc
-                    -> /tmp/fasel.pyc_dis, /tmp/foo.pyc_dis
-                  uncompyle6 -o /tmp bla/fasel.pyc bar/foo.pyc
-                    -> /tmp/bla/fasel.pyc_dis, /tmp/bar/foo.pyc_dis
-                  uncompyle6 -o /tmp /usr/lib/python1.5
-                    -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis
-  --compile | -c <python-file>
-                attempts a decompilation after compiling <python-file>
-  -d            print timestamps
-  -p <integer>  use <integer> number of processes
-  -r            recurse directories looking for .pyc and .pyo files
-  --fragments   use fragments deparser
-  --verify      compare generated source with input byte-code
-  --verify-run  compile generated source, run it and check exit code
-  --syntax-verify compile generated source
-  --linemaps    generated line number correspondencies between byte-code
-                and generated source output
-  --encoding  <encoding>
-                use <encoding> in generated source according to pep-0263
-  --help        show this message
-
-Debugging Options:
-  --asm     | -a        include byte-code       (disables --verify)
-  --grammar | -g        show matching grammar
-  --tree={before|after}
-  -t {before|after}     include syntax before (or after) tree transformation
-                        (disables --verify)
-  --tree++ | -T         add template rules to --tree=before when possible
-
-Extensions of generated files:
-  '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
-    + '_unverified'       successfully decompile but --verify failed
-    + '_failed'           decompile failed (contact author for enhancement)
-""" % ((program,) * 5)
-
-program = 'uncompyle6'
-
-from uncompyle6 import verify
 from uncompyle6.main import main, status_msg
+from uncompyle6.verify import VerifyCmpError
 from uncompyle6.version import __version__
 
+program = "uncompyle6"
+
+
 def usage():
     print(__doc__)
     sys.exit(1)
 
 
-def main_bin():
-    if not (sys.version_info[0:2] in ((2, 6), (2, 7), (3, 0),
-                                      (3, 1), (3, 2), (3, 3),
-                                      (3, 4), (3, 5), (3, 6),
-                                      (3, 7), (3, 8), (3, 9), (3, 10)
-        )):
-        print(
-            f"Error: {program} can decompile only bytecode from Python 3.7"
-            f""" to 3.8.\n\tYou have version: {version_tuple_to_str()}."""
-        )
-        sys.exit(-1)
+# __doc__ = """
+# Usage:
+#   %s [OPTIONS]... [ FILE | DIR]...
+#   %s [--help | -h | --V | --version]
+
+# Examples:
+#   %s      foo.pyc bar.pyc       # decompile foo.pyc, bar.pyc to stdout
+#   %s -o . foo.pyc bar.pyc       # decompile to ./foo.pyc_dis and ./bar.pyc_dis
+#   %s -o /tmp /usr/lib/python1.5 # decompile whole library
+
+# Options:
+#   -o <path>     output decompiled files to this path:
+#                 if multiple input files are decompiled, the common prefix
+#                 is stripped from these names and the remainder appended to
+#                 <path>
+#                   uncompyle6 -o /tmp bla/fasel.pyc bla/foo.pyc
+#                     -> /tmp/fasel.pyc_dis, /tmp/foo.pyc_dis
+#                   uncompyle6 -o /tmp bla/fasel.pyc bar/foo.pyc
+#                     -> /tmp/bla/fasel.pyc_dis, /tmp/bar/foo.pyc_dis
+#                   uncompyle6 -o /tmp /usr/lib/python1.5
+#                     -> /tmp/smtplib.pyc_dis ... /tmp/lib-tk/FixTk.pyc_dis
+#   --compile | -c <python-file>
+#                 attempts a decompilation after compiling <python-file>
+#   -d            print timestamps
+#   -p <integer>  use <integer> number of processes
+#   -r            recurse directories looking for .pyc and .pyo files
+#   --fragments   use fragments deparser
+#   --verify      compare generated source with input byte-code
+#   --verify-run  compile generated source, run it and check exit code
+#   --syntax-verify compile generated source
+#   --linemaps    generated line number correspondencies between byte-code
+#                 and generated source output
+#   --encoding  <encoding>
+#                 use <encoding> in generated source according to pep-0263
+#   --help        show this message
+
+# Debugging Options:
+#   --asm     | -a        include byte-code       (disables --verify)
+#   --grammar | -g        show matching grammar
+#   --tree={before|after}
+#   -t {before|after}     include syntax before (or after) tree transformation
+#                         (disables --verify)
+#   --tree++ | -T         add template rules to --tree=before when possible
+
+# Extensions of generated files:
+#   '.pyc_dis' '.pyo_dis'   successfully decompiled (and verified if --verify)
+#     + '_unverified'       successfully decompile but --verify failed
+#     + '_failed'           decompile failed (contact author for enhancement)
+# """ % (
+#     (program,) * 5
+# )
+
+
+@click.command()
+@click.option(
+    "--asm++/--no-asm++",
+    "-A",
+    "asm_plus",
+    default=False,
+    help="show xdis assembler and tokenized assembler",
+)
+@click.option("--asm/--no-asm", "-a", default=False)
+@click.option("--grammar/--no-grammar", "-g", "show_grammar", default=False)
+@click.option("--tree/--no-tree", "-t", default=False)
+@click.option(
+    "--tree++/--no-tree++",
+    "-T",
+    "tree_plus",
+    default=False,
+    help="show parse tree and Abstract Syntax Tree",
+)
+@click.option(
+    "--linemaps/--no-linemaps",
+    default=False,
+    help="show line number correspondencies between byte-code "
+    "and generated source output",
+)
+@click.option(
+    "--verify",
+    type=click.Choice(["run", "syntax"]),
+    default=None,
+)
+@click.option(
+    "--recurse/--no-recurse",
+    "-r",
+    "recurse_dirs",
+    default=False,
+)
+@click.option(
+    "--output",
+    "-o",
+    "outfile",
+    type=click.Path(
+        exists=True, file_okay=True, dir_okay=True, writable=True, resolve_path=True
+    ),
+    required=False,
+)
+@click.version_option(version=__version__)
+@click.option(
+    "--start-offset",
+    "start_offset",
+    default=0,
+    help="start decomplation at offset; default is 0 or the starting offset.",
+)
+@click.version_option(version=__version__)
+@click.option(
+    "--stop-offset",
+    "stop_offset",
+    default=-1,
+    help="stop decomplation when seeing an offset greater or equal to this; default is "
+    "-1 which indicates no stopping point.",
+)
+@click.argument("files", nargs=-1, type=click.Path(readable=True), required=True)
+def main_bin(
+    asm: bool,
+    asm_plus: bool,
+    show_grammar,
+    tree: bool,
+    tree_plus: bool,
+    linemaps: bool,
+    verify,
+    recurse_dirs: bool,
+    outfile,
+    start_offset: int,
+    stop_offset: int,
+    files,
+):
+    """
+    Cross Python bytecode decompiler for Python bytecode up to Python 3.8.
+    """
+
+    version_tuple = sys.version_info[0:2]
+    if not ((3, 0) <= version_tuple < (3, 3)):
+        if version_tuple > (3, 3):
+            print(
+                "This version of the {program} is tailored for Python 3.0 to 3.2.\n"
+                "It may run on other versions, but there are problems, switch to code "
+                "from another branch.\n"
+                "You have version: %s." % version_tuple_to_str()
+            )
+        else:
+            print(
+                "Error: This version of the {program} runs from Python 3.0 to 3.2.\n"
+                "You need another branch of this code for other Python versions."
+                " \n\tYou have version: %s." % version_tuple_to_str()
+            )
+            sys.exit(-1)
 
-    do_verify = recurse_dirs = False
     numproc = 0
-    outfile = '-'
+    out_base = None
+
     out_base = None
     source_paths = []
     timestamp = False
     timestampfmt = "# %Y.%m.%d %H:%M:%S %Z"
+    pyc_paths = files
 
-    try:
-        opts, pyc_paths = getopt.getopt(sys.argv[1:], 'hac:gtTdrVo:p:',
-                                    'help asm compile= grammar linemaps recurse '
-                                    'timestamp tree= tree+ '
-                                    'fragments verify verify-run version '
-                                    'syntax-verify '
-                                    'showgrammar encoding='.split(' '))
-    except getopt.GetoptError as e:
-        print('%s: %s' % (os.path.basename(sys.argv[0]), e),  file=sys.stderr)
-        sys.exit(-1)
-
-    options = {}
-    for opt, val in opts:
-        if opt in ('-h', '--help'):
-            print(__doc__)
-            sys.exit(0)
-        elif opt in ('-V', '--version'):
-            print("%s %s" % (program, __version__))
-            sys.exit(0)
-        elif opt == '--verify':
-            options['do_verify'] = 'strong'
-        elif opt == '--syntax-verify':
-            options['do_verify'] = 'weak'
-        elif opt == '--fragments':
-            options['do_fragments'] = True
-        elif opt == '--verify-run':
-            options['do_verify'] = 'verify-run'
-        elif opt == '--linemaps':
-            options['do_linemaps'] = True
-        elif opt in ('--asm', '-a'):
-            options['showasm'] = 'after'
-            options['do_verify'] = None
-        elif opt in ('--tree', '-t'):
-            if 'showast' not in options:
-                options['showast'] = {}
-            if val == 'before':
-                options['showast'][val] = True
-            elif val == 'after':
-                options['showast'][val] = True
-            else:
-                options['showast']['before'] = True
-            options['do_verify'] = None
-        elif opt in ('--tree+', '-T'):
-            if 'showast' not in options:
-                options['showast'] = {}
-            options['showast']['after'] = True
-            options['showast']['before'] = True
-            options['do_verify'] = None
-        elif opt in ('--grammar', '-g'):
-            options['showgrammar'] = True
-        elif opt == '-o':
-            outfile = val
-        elif opt in ('--timestamp', '-d'):
-            timestamp = True
-        elif opt in ('--compile', '-c'):
-            source_paths.append(val)
-        elif opt == '-p':
-            numproc = int(val)
-        elif opt in ('--recurse', '-r'):
-            recurse_dirs = True
-        elif opt == '--encoding':
-            options['source_encoding'] = val
-        else:
-            print(opt, file=sys.stderr)
-            usage()
-
-    # expand directory if specified
+    # Expand directory if "recurse" was specified.
     if recurse_dirs:
         expanded_files = []
         for f in pyc_paths:
             if os.path.isdir(f):
                 for root, _, dir_files in os.walk(f):
                     for df in dir_files:
-                        if df.endswith('.pyc') or df.endswith('.pyo'):
+                        if df.endswith(".pyc") or df.endswith(".pyo"):
                             expanded_files.append(os.path.join(root, df))
         pyc_paths = expanded_files
 
@@ -175,38 +200,58 @@ def main_bin():
     if src_base[-1:] != os.sep:
         src_base = os.path.dirname(src_base)
     if src_base:
-        sb_len = len( os.path.join(src_base, '') )
+        sb_len = len(os.path.join(src_base, ""))
         pyc_paths = [f[sb_len:] for f in pyc_paths]
 
     if not pyc_paths and not source_paths:
         print("No input files given to decompile", file=sys.stderr)
         usage()
 
-    if outfile == '-':
-        outfile = None # use stdout
+    if outfile == "-":
+        outfile = None  # use stdout
     elif outfile and os.path.isdir(outfile):
-        out_base = outfile; outfile = None
+        out_base = outfile
+        outfile = None
     elif outfile and len(pyc_paths) > 1:
-        out_base = outfile; outfile = None
+        out_base = outfile
+        outfile = None
+
+    # A second -a turns show_asm="after" into show_asm="before"
+    if asm_plus or asm:
+        asm_opt = "both" if asm_plus else "after"
+    else:
+        asm_opt = None
 
     if timestamp:
         print(time.strftime(timestampfmt))
 
     if numproc <= 1:
+        show_ast = {"before": tree or tree_plus, "after": tree_plus}
         try:
-            result = main(src_base, out_base, pyc_paths, source_paths, outfile,
-                          **options)
-            result = [options.get('do_verify', None)] + list(result)
+            result = main(
+                src_base,
+                out_base,
+                pyc_paths,
+                source_paths,
+                outfile,
+                showasm=asm_opt,
+                showgrammar=show_grammar,
+                showast=show_ast,
+                do_verify=verify,
+                do_linemaps=linemaps,
+                start_offset=start_offset,
+                stop_offset=stop_offset,
+            )
             if len(pyc_paths) > 1:
                 mess = status_msg(*result)
-                print('# ' + mess)
+                print("# " + mess)
                 pass
         except ImportError as e:
             print(str(e))
             sys.exit(2)
-        except (KeyboardInterrupt):
+        except KeyboardInterrupt:
             pass
-        except verify.VerifyCmpError:
+        except VerifyCmpError:
             raise
     else:
         from multiprocessing import Process, Queue
@@ -216,7 +261,7 @@ def main_bin():
         except ImportError:
             from queue import Empty
 
-        fqueue = Queue(len(pyc_paths)+numproc)
+        fqueue = Queue(len(pyc_paths) + numproc)
         for f in pyc_paths:
             fqueue.put(f)
         for i in range(numproc):
@@ -224,15 +269,21 @@ def main_bin():
 
         rqueue = Queue(numproc)
 
+        tot_files = okay_files = failed_files = verify_failed_files = 0
+
         def process_func():
+            (tot_files, okay_files, failed_files, verify_failed_files) = (
+                0,
+                0,
+                0,
+                0,
+            )
             try:
-                (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
                 while 1:
                     f = fqueue.get()
                     if f is None:
                         break
-                    (t, o, f, v) = \
-                      main(src_base, out_base, [f], [], outfile, **options)
+                    (t, o, f, v) = main(src_base, out_base, [f], [], outfile)
                     tot_files += t
                     okay_files += o
                     failed_files += f
@@ -249,7 +300,12 @@ def main_bin():
             for p in procs:
                 p.join()
             try:
-                (tot_files, okay_files, failed_files, verify_failed_files) = (0, 0, 0, 0)
+                (tot_files, okay_files, failed_files, verify_failed_files) = (
+                    0,
+                    0,
+                    0,
+                    0,
+                )
                 while True:
                     (t, o, f, v) = rqueue.get(False)
                     tot_files += t
@@ -258,8 +314,10 @@ def main_bin():
                     verify_failed_files += v
             except Empty:
                 pass
-            print('# decompiled %i files: %i okay, %i failed, %i verify failed' %
-                  (tot_files, okay_files, failed_files, verify_failed_files))
+            print(
+                "# decompiled %i files: %i okay, %i failed, %i verify failed"
+                % (tot_files, okay_files, failed_files, verify_failed_files)
+            )
         except (KeyboardInterrupt, OSError):
             pass
 
@@ -268,5 +326,6 @@ def main_bin():
 
     return
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main_bin()

uncompyle6/main.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2022 Rocky Bernstein <rocky@gnu.org>
+# Copyright (C) 2018-2024 Rocky Bernstein <rocky@gnu.org>
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -13,40 +13,59 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from typing import Any, Tuple
-import datetime, py_compile, os, sys
+import ast
+import datetime
+import os
+import os.path as osp
+import py_compile
+import subprocess
+import sys
+import tempfile
 
 from xdis import iscode
+from xdis.load import load_module
 from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE, version_tuple_to_str
+
 from uncompyle6.code_fns import check_object_path
-from uncompyle6.semantics import pysource
-from uncompyle6.semantics.pysource import PARSER_DEFAULT_DEBUG
 from uncompyle6.parser import ParserError
+from uncompyle6.semantics import pysource
+from uncompyle6.semantics.fragments import code_deparse as code_deparse_fragments
+from uncompyle6.semantics.linemap import deparse_code_with_map
+from uncompyle6.semantics.pysource import PARSER_DEFAULT_DEBUG, code_deparse
 from uncompyle6.version import __version__
 
 # from uncompyle6.linenumbers import line_number_mapping
 
-from uncompyle6.semantics.pysource import code_deparse
-from uncompyle6.semantics.fragments import code_deparse as code_deparse_fragments
-from uncompyle6.semantics.linemap import deparse_code_with_map
 
-from xdis.load import load_module
-
-def _get_outstream(outfile: str) -> Any:
-    dir = os.path.dirname(outfile)
+def _get_outstream(outfile):
+    """
+    Return an opened output file descriptor for ``outfile``.
+    """
+    dir_name = osp.dirname(outfile)
     failed_file = outfile + "_failed"
-    if os.path.exists(failed_file):
+    if osp.exists(failed_file):
         os.remove(failed_file)
     try:
-        os.makedirs(dir)
+        os.makedirs(dir_name)
     except OSError:
         pass
     return open(outfile, mode="w", encoding="utf-8")
 
 
+def syntax_check(filename: str) -> bool:
+    with open(filename) as f:
+        source = f.read()
+    valid = True
+    try:
+        ast.parse(source)
+    except SyntaxError:
+        valid = False
+    return valid
+
+
 def decompile(
     co,
-    bytecode_version: Tuple[int] = PYTHON_VERSION_TRIPLE,
+    bytecode_version=PYTHON_VERSION_TRIPLE,
     out=sys.stdout,
     showasm=None,
     showast={},
@@ -55,16 +74,18 @@ def decompile(
     source_encoding=None,
     code_objects={},
     source_size=None,
-    is_pypy=None,
+    is_pypy: bool = False,
     magic_int=None,
     mapstream=None,
     do_fragments=False,
     compile_mode="exec",
-) -> Any:
+    start_offset: int = 0,
+    stop_offset: int = -1,
+):
     """
     ingests and deparses a given code block 'co'
 
-    if `bytecode_version` is None, use the current Python intepreter
+    if `bytecode_version` is None, use the current Python interpreter
     version.
 
     Caller is responsible for closing `out` and `mapstream`
@@ -79,7 +100,7 @@ def decompile(
         s += "\n"
         real_out.write(s)
 
-    assert iscode(co), f"""{co} does not smell like code"""
+    assert iscode(co), "%s does not smell like code" % co
 
     co_pypy_str = "PyPy " if is_pypy else ""
     run_pypy_str = "PyPy " if IS_PYPY else ""
@@ -105,36 +126,34 @@ def decompile(
     if source_size:
         write("# Size of source mod 2**32: %d bytes" % source_size)
 
-    # maybe a second -a will do before as well
-    asm = "after" if showasm else None
-
     grammar = dict(PARSER_DEFAULT_DEBUG)
     if showgrammar:
         grammar["reduce"] = True
 
-    debug_opts = {"asm": asm, "tree": showast, "grammar": grammar}
+    debug_opts = {"asm": showasm, "tree": showast, "grammar": grammar}
 
     try:
         if mapstream:
             if isinstance(mapstream, str):
                 mapstream = _get_outstream(mapstream)
 
+            debug_opts = {"asm": showasm, "tree": showast, "grammar": grammar}
+
             deparsed = deparse_code_with_map(
-                bytecode_version,
-                co,
-                out,
-                showasm,
-                showast,
-                showgrammar,
+                co=co,
+                out=out,
+                version=bytecode_version,
                 code_objects=code_objects,
                 is_pypy=is_pypy,
+                debug_opts=debug_opts,
             )
             header_count = 3 + len(sys_version_lines)
-            linemap = [
-                (line_no, deparsed.source_linemap[line_no] + header_count)
-                for line_no in sorted(deparsed.source_linemap.keys())
-            ]
-            mapstream.write("\n\n# %s\n" % linemap)
+            if deparsed is not None:
+                linemap = [
+                    (line_no, deparsed.source_linemap[line_no] + header_count)
+                    for line_no in sorted(deparsed.source_linemap.keys())
+                ]
+                mapstream.write("\n\n# %s\n" % linemap)
         else:
             if do_fragments:
                 deparse_fn = code_deparse_fragments
@@ -144,18 +163,21 @@ def decompile(
                 co,
                 out,
                 bytecode_version,
-                debug_opts=debug_opts,
                 is_pypy=is_pypy,
+                debug_opts=debug_opts,
                 compile_mode=compile_mode,
+                start_offset=start_offset,
+                stop_offset=stop_offset,
             )
             pass
+        real_out.write("\n")
         return deparsed
     except pysource.SourceWalkerError as e:
         # deparsing failed
         raise pysource.SourceWalkerError(str(e))
 
 
-def compile_file(source_path: str) -> str:
+def compile_file(source_path):
     if source_path.endswith(".py"):
         basename = source_path[:-3]
     else:
@@ -180,7 +202,9 @@ def decompile_file(
     source_encoding=None,
     mapstream=None,
     do_fragments=False,
-) -> Any:
+    start_offset=0,
+    stop_offset=-1,
+):
     """
     decompile Python byte-code file (.pyc). Return objects to
     all of the deparsed objects found in `filename`.
@@ -188,7 +212,7 @@ def decompile_file(
 
     filename = check_object_path(filename)
     code_objects = {}
-    (version, timestamp, magic_int, co, is_pypy, source_size, sip_hash) = load_module(
+    version, timestamp, magic_int, co, is_pypy, source_size, _ = load_module(
         filename, code_objects
     )
 
@@ -209,6 +233,8 @@ def decompile_file(
                     is_pypy=is_pypy,
                     magic_int=magic_int,
                     mapstream=mapstream,
+                    start_offset=start_offset,
+                    stop_offset=stop_offset,
                 ),
             )
     else:
@@ -229,28 +255,30 @@ def decompile_file(
                 mapstream=mapstream,
                 do_fragments=do_fragments,
                 compile_mode="exec",
+                start_offset=start_offset,
+                stop_offset=stop_offset,
             )
         ]
-    co = None
     return deparsed
 
 
 # FIXME: combine into an options parameter
 def main(
     in_base: str,
-    out_base: str,
+    out_base,
     compiled_files: list,
     source_files: list,
     outfile=None,
     showasm=None,
     showast={},
-    do_verify=False,
-    showgrammar=False,
+    do_verify=None,
+    showgrammar: bool = False,
     source_encoding=None,
-    raise_on_error=False,
     do_linemaps=False,
     do_fragments=False,
-) -> Tuple[int, int, int, int]:
+    start_offset: int = 0,
+    stop_offset: int = -1,
+):
     """
     in_base	base directory for input files
     out_base	base directory for output files (ignored when
@@ -262,7 +290,8 @@ def main(
     - files below out_base	out_base=...
     - stdout			out_base=None, outfile=None
     """
-    tot_files = okay_files = failed_files = verify_failed_files = 0
+    tot_files = okay_files = failed_files = 0
+    verify_failed_files = 0 if do_verify else 0
     current_outfile = outfile
     linemap_stream = None
 
@@ -270,9 +299,9 @@ def main(
         compiled_files.append(compile_file(source_path))
 
     for filename in compiled_files:
-        infile = os.path.join(in_base, filename)
+        infile = osp.join(in_base, filename)
         # print("XXX", infile)
-        if not os.path.exists(infile):
+        if not osp.exists(infile):
             sys.stderr.write("File '%s' doesn't exist. Skipped\n" % infile)
             continue
 
@@ -285,14 +314,19 @@ def main(
         if outfile:  # outfile was given as parameter
             outstream = _get_outstream(outfile)
         elif out_base is None:
-            outstream = sys.stdout
+            out_base = tempfile.mkdtemp(prefix="py-dis-")
+            if do_verify and filename.endswith(".pyc"):
+                current_outfile = osp.join(out_base, filename[0:-1])
+                outstream = open(current_outfile, "w")
+            else:
+                outstream = sys.stdout
             if do_linemaps:
                 linemap_stream = sys.stdout
         else:
             if filename.endswith(".pyc"):
-                current_outfile = os.path.join(out_base, filename[0:-1])
+                current_outfile = osp.join(out_base, filename[0:-1])
             else:
-                current_outfile = os.path.join(out_base, filename) + "_dis"
+                current_outfile = osp.join(out_base, filename) + "_dis"
                 pass
             pass
 
@@ -300,9 +334,9 @@ def main(
 
         # print(current_outfile, file=sys.stderr)
 
-        # Try to uncompile the input file
+        # Try to decompile the input file.
         try:
-            deparsed = decompile_file(
+            deparsed_objects = decompile_file(
                 infile,
                 outstream,
                 showasm,
@@ -311,11 +345,13 @@ def main(
                 source_encoding,
                 linemap_stream,
                 do_fragments,
+                start_offset,
+                stop_offset,
             )
             if do_fragments:
-                for d in deparsed:
+                for deparsed_object in deparsed_objects:
                     last_mod = None
-                    offsets = d.offsets
+                    offsets = deparsed_object.offsets
                     for e in sorted(
                         [k for k in offsets.keys() if isinstance(k[1], int)]
                     ):
@@ -324,16 +360,52 @@ def main(
                             outstream.write("%s\n%s\n%s\n" % (line, e[0], line))
                         last_mod = e[0]
                         info = offsets[e]
-                        extractInfo = d.extract_node_info(info)
+                        extract_info = deparsed_object.extract_node_info(info)
                         outstream.write("%s" % info.node.format().strip() + "\n")
-                        outstream.write(extractInfo.selectedLine + "\n")
-                        outstream.write(extractInfo.markerLine + "\n\n")
+                        outstream.write(extract_info.selectedLine + "\n")
+                        outstream.write(extract_info.markerLine + "\n\n")
                     pass
+
+            if do_verify:
+                for deparsed_object in deparsed_objects:
+                    deparsed_object.f.close()
+                    if PYTHON_VERSION_TRIPLE[:2] != deparsed_object.version[:2]:
+                        sys.stdout.write(
+                            "\n# skipping running %s; it is %s and we are %s"
+                            % (
+                                deparsed_object.f.name,
+                                version_tuple_to_str(deparsed_object.version, end=2),
+                                version_tuple_to_str(PYTHON_VERSION_TRIPLE, end=2),
+                            )
+                        )
+                    else:
+                        check_type = "syntax check"
+                        if do_verify == "run":
+                            check_type = "run"
+                            return_code = subprocess.call(
+                                [sys.executable, deparsed_object.f.name],
+                                stdout=sys.stdout,
+                                stderr=sys.stderr,
+                            )
+                            valid = return_code == 0
+                            if not valid:
+                                sys.stderr.write("Got return code %d\n" % return_code)
+                        else:
+                            valid = syntax_check(deparsed_object.f.name)
+
+                        if not valid:
+                            verify_failed_files += 1
+                            sys.stderr.write(
+                                "\n# %s failed on file %s\n"
+                                % (check_type, deparsed_object.f.name)
+                            )
+
+                    # sys.stderr.write("Ran %\n" % deparsed_object.f.name)
                 pass
             tot_files += 1
         except (ValueError, SyntaxError, ParserError, pysource.SourceWalkerError) as e:
             sys.stdout.write("\n")
-            sys.stderr.write(f"\n# file {infile}\n# {e}\n")
+            sys.stderr.write("\n# file %s\n# %s\n" % (infile, e))
             failed_files += 1
             tot_files += 1
         except KeyboardInterrupt:
@@ -341,10 +413,10 @@ def main(
                 outstream.close()
                 os.remove(outfile)
             sys.stdout.write("\n")
-            sys.stderr.write(f"\nLast file: {infile}   ")
+            sys.stderr.write("\nLast file: %s   " % (infile))
             raise
         except RuntimeError as e:
-            sys.stdout.write(f"\n{str(e)}\n")
+            sys.stdout.write("\n%s\n" % str(e))
             if str(e).startswith("Unsupported Python"):
                 sys.stdout.write("\n")
                 sys.stderr.write(
@@ -375,7 +447,7 @@ def main(
                 okay_files += 1
                 if not current_outfile:
                     mess = "\n# okay decompiling"
-                    # mem_usage = __memUsage()
+                    # mem_usage = __mem_usage()
                     print(mess, infile)
         if current_outfile:
             sys.stdout.write(
@@ -383,7 +455,6 @@ def main(
                 % (
                     infile,
                     status_msg(
-                        do_verify,
                         tot_files,
                         okay_files,
                         failed_files,
@@ -394,37 +465,36 @@ def main(
             try:
                 # FIXME: Something is weird with Pypy here
                 sys.stdout.flush()
-            except:
+            except Exception:
                 pass
     if current_outfile:
         sys.stdout.write("\n")
         try:
             # FIXME: Something is weird with Pypy here
             sys.stdout.flush()
-        except:
+        except Exception:
             pass
         pass
-    return (tot_files, okay_files, failed_files, verify_failed_files)
+    return tot_files, okay_files, failed_files, verify_failed_files
 
 
 # ---- main ----
 
 if sys.platform.startswith("linux") and os.uname()[2][:2] in ["2.", "3.", "4."]:
 
-    def __memUsage():
+    def __mem_sage():
         mi = open("/proc/self/stat", "r")
         mu = mi.readline().split()[22]
         mi.close()
         return int(mu) / 1000000
 
-
 else:
 
-    def __memUsage():
+    def __mem_usage():
         return ""
 
 
-def status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_files):
+def status_msg(tot_files, okay_files, failed_files, verify_failed_files):
     if tot_files == 1:
         if failed_files:
             return "\n# decompile failed"
@@ -434,5 +504,9 @@ def status_msg(do_verify, tot_files, okay_files, failed_files, verify_failed_fil
             return "\n# Successfully decompiled file"
             pass
         pass
-    mess = f"decompiled {tot_files} files: {okay_files} okay, {failed_files} failed"
+    mess = "decompiled %i files: %i okay, %i failed" % (
+        tot_files,
+        okay_files,
+        failed_files,
+    )
     return mess

uncompyle6/parser.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2022 Rocky Bernstein
+#  Copyright (c) 2015-2024 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -21,10 +21,11 @@ Common uncompyle6 parser routines.
 
 import sys
 
-from spark_parser import GenericASTBuilder, DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
-from uncompyle6.show import maybe_show_asm
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG, GenericASTBuilder
 from xdis import iscode
 
+from uncompyle6.show import maybe_show_asm
+
 
 class ParserError(Exception):
     def __init__(self, token, offset, debug=PARSER_DEFAULT_DEBUG):
@@ -44,8 +45,8 @@ def nop_func(self, args):
 
 
 class PythonParser(GenericASTBuilder):
-    def __init__(self, SyntaxTree, start, debug):
-        super(PythonParser, self).__init__(SyntaxTree, start, debug)
+    def __init__(self, syntax_tree_class, start, debug):
+        super(PythonParser, self).__init__(syntax_tree_class, start, debug)
         # FIXME: customize per python parser version
 
         # These are the non-terminals we should collect into a list.
@@ -91,7 +92,14 @@ class PythonParser(GenericASTBuilder):
         # singleton reduction that we can simplify. It also happens to be optional
         # in its other derivation
         self.optional_nt |= frozenset(
-            ("come_froms", "suite_stmts", "l_stmts_opt", "c_stmts_opt", "stmts_opt", "stmt")
+            (
+                "come_froms",
+                "suite_stmts",
+                "l_stmts_opt",
+                "c_stmts_opt",
+                "stmts_opt",
+                "stmt",
+            )
         )
 
         # Reduce singleton reductions in these nonterminals:
@@ -103,6 +111,7 @@ class PythonParser(GenericASTBuilder):
         )
         # Instructions filled in from scanner
         self.insts = []
+        self.version = tuple()
 
     def ast_first_offset(self, ast):
         if hasattr(ast, "offset"):
@@ -112,10 +121,10 @@ class PythonParser(GenericASTBuilder):
 
     def add_unique_rule(self, rule, opname, arg_count, customize):
         """Add rule to grammar, but only if it hasn't been added previously
-           opname and stack_count are used in the customize() semantic
-           the actions to add the semantic action rule. Stack_count is
-           used in custom opcodes like MAKE_FUNCTION to indicate how
-           many arguments it has. Often it is not used.
+        opname and stack_count are used in the customize() semantic
+        the actions to add the semantic action rule. Stack_count is
+        used in custom opcodes like MAKE_FUNCTION to indicate how
+        many arguments it has. Often it is not used.
         """
         if rule not in self.new_rules:
             # print("XXX ", rule) # debug
@@ -151,9 +160,9 @@ class PythonParser(GenericASTBuilder):
         Remove recursive references to allow garbage
         collector to collect this object.
         """
-        for dict in (self.rule2func, self.rules, self.rule2name):
-            for i in list(dict.keys()):
-                dict[i] = None
+        for rule_dict in (self.rule2func, self.rules, self.rule2name):
+            for i in list(rule_dict.keys()):
+                rule_dict[i] = None
         for i in dir(self):
             setattr(self, i, None)
 
@@ -164,11 +173,11 @@ class PythonParser(GenericASTBuilder):
 
         def fix(c):
             s = str(c)
-            last_token_pos = s.find("_")
-            if last_token_pos == -1:
+            token_pos = s.find("_")
+            if token_pos == -1:
                 return s
             else:
-                return s[:last_token_pos]
+                return s[:token_pos]
 
         prefix = ""
         if parent and tokens:
@@ -220,9 +229,11 @@ class PythonParser(GenericASTBuilder):
 
         This appears in CALL_FUNCTION or CALL_METHOD (PyPy) tokens
         """
-        # Low byte indicates number of positional paramters,
+        # Low byte indicates number of positional parameters,
         # high byte number of keyword parameters
-        assert token.kind.startswith("CALL_FUNCTION") or token.kind.startswith("CALL_METHOD")
+        assert token.kind.startswith("CALL_FUNCTION") or token.kind.startswith(
+            "CALL_METHOD"
+        )
         args_pos = token.attr & 0xFF
         args_kw = (token.attr >> 8) & 0xFF
         return args_pos, args_kw
@@ -267,13 +278,13 @@ class PythonParser(GenericASTBuilder):
         print(children)
         return GenericASTBuilder.ambiguity(self, children)
 
-    def resolve(self, list):
-        if len(list) == 2 and "function_def" in list and "assign" in list:
+    def resolve(self, rule: list):
+        if len(rule) == 2 and "function_def" in rule and "assign" in rule:
             return "function_def"
-        if "grammar" in list and "expr" in list:
+        if "grammar" in rule and "expr" in rule:
             return "expr"
-        # print >> sys.stderr, 'resolve', str(list)
-        return GenericASTBuilder.resolve(self, list)
+        # print >> sys.stderr, 'resolve', str(rule)
+        return GenericASTBuilder.resolve(self, rule)
 
     ###############################################
     #  Common Python 2 and Python 3 grammar rules #
@@ -303,6 +314,9 @@ class PythonParser(GenericASTBuilder):
         c_stmts ::= lastc_stmt
         c_stmts ::= continues
 
+        ending_return  ::= RETURN_VALUE RETURN_LAST
+        ending_return  ::= RETURN_VALUE_LAMBDA LAMBDA_MARKER
+
         lastc_stmt ::= iflaststmt
         lastc_stmt ::= forelselaststmt
         lastc_stmt ::= ifelsestmtc
@@ -360,7 +374,7 @@ class PythonParser(GenericASTBuilder):
         stmt ::= tryelsestmt
         stmt ::= tryfinallystmt
         stmt ::= with
-        stmt ::= withasstmt
+        stmt ::= with_as
 
         stmt   ::= delete
         delete ::= DELETE_FAST
@@ -596,11 +610,12 @@ class PythonParser(GenericASTBuilder):
         compare        ::= compare_single
         compare_single ::= expr expr COMPARE_OP
 
-        # A compare_chained is two comparisions like x <= y <= z
-        compare_chained  ::= expr compare_chained1 ROT_TWO POP_TOP _come_froms
-        compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD
+        # A compare_chained is two comparisons, as in: x <= y <= z
+        compare_chained       ::= expr compared_chained_middle ROT_TWO POP_TOP
+                                  _come_froms
+        compare_chained_right ::= expr COMPARE_OP JUMP_FORWARD
 
-        # Non-null kvlist items are broken out in the indiviual grammars
+        # Non-null kvlist items are broken out in the individual grammars
         kvlist ::=
 
         # Positional arguments in make_function
@@ -667,7 +682,7 @@ def get_python_parser(
                 if compile_mode == "exec":
                     p = parse10.Python10Parser(debug_parser)
                 else:
-                    p = parse10.Python01ParserSingle(debug_parser)
+                    p = parse10.Python10ParserSingle(debug_parser)
             elif version == (1, 1):
                 import uncompyle6.parsers.parse11 as parse11
 
@@ -873,6 +888,7 @@ def python_parser(
     :param showasm:         Flag which determines whether the disassembled and
                             ingested code is written to sys.stdout or not.
     :param parser_debug:    dict containing debug flags for the spark parser.
+    :param is_pypy:         True if we are running PyPY
 
     :return: Abstract syntax tree representation of the code object.
     """
@@ -901,7 +917,7 @@ def python_parser(
 if __name__ == "__main__":
 
     def parse_test(co):
-        from xdis import PYTHON_VERSION_TRIPLE, IS_PYPY
+        from xdis import IS_PYPY, PYTHON_VERSION_TRIPLE
 
         ast = python_parser(PYTHON_VERSION_TRIPLE, co, showasm=True, is_pypy=IS_PYPY)
         print(ast)

uncompyle6/parsers/parse13.py

@@ -1,11 +1,12 @@
-#  Copyright (c) 2018 Rocky Bernstein
+#  Copyright (c) 2018, 2023 Rocky Bernstein
 
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse14 import Python14Parser
 
-class Python13Parser(Python14Parser):
 
+class Python13Parser(Python14Parser):
     def p_misc13(self, args):
         """
         # Nothing here yet, but will need to add LOAD_GLOBALS
@@ -24,7 +25,6 @@ class Python13Parser(Python14Parser):
     #     """)
     #     self.check_reduce['doc_junk'] = 'tokens'
 
-
     # def reduce_is_invalid(self, rule, ast, tokens, first, last):
     #     invalid = super(Python14Parser,
     #                     self).reduce_is_invalid(rule, ast,
@@ -35,11 +35,11 @@ class Python13Parser(Python14Parser):
     #         return not isinstance(tokens[first].pattr, str)
 
 
-
 class Python13ParserSingle(Python13Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Check grammar
     p = Python13Parser()
     p.check_grammar()

uncompyle6/parsers/parse14.py

@@ -64,7 +64,7 @@ class Python14Parser(Python15Parser):
 
             if opname_base == "UNPACK_VARARG":
                 if token.attr > 1:
-                    self.addRule(f"star_args ::= RESERVE_FAST {opname} args_store", nop_func)
+                    self.addRule("star_args ::= RESERVE_FAST %s args_store" % opname, nop_func)
 
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):

uncompyle6/parsers/parse2.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2021 Rocky Bernstein
+#  Copyright (c) 2015-2021, 2024 Rocky Bernstein
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
 #  Copyright (c) 1999 John Aycock
@@ -27,11 +27,12 @@ that a later phase can turn into a sequence of ASCII text.
 
 from __future__ import print_function
 
-from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
-from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
-from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 
+from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
+from uncompyle6.parsers.reducecheck import except_handler_else, ifelsestmt, tryelsestmt
+from uncompyle6.parsers.treenode import SyntaxTree
+
 
 class Python2Parser(PythonParser):
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -405,7 +406,6 @@ class Python2Parser(PythonParser):
                 "CALL_FUNCTION_VAR_KW",
                 "CALL_FUNCTION_KW",
             ):
-
                 args_pos, args_kw = self.get_pos_kw(token)
 
                 # number of apply equiv arguments:
@@ -526,7 +526,7 @@ class Python2Parser(PythonParser):
                 custom_seen_ops.add(opname)
                 continue
             elif opname == "LOAD_LISTCOMP":
-                self.addRule("expr ::= listcomp", nop_func)
+                self.addRule("expr ::= list_comp", nop_func)
                 custom_seen_ops.add(opname)
                 continue
             elif opname == "LOAD_SETCOMP":

uncompyle6/parsers/parse24.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2018, 2020, 2022 Rocky Bernstein
+#  Copyright (c) 2016-2018, 2020, 2022-2024 Rocky Bernstein
 """
 spark grammar differences over Python2.5 for Python 2.4.
 """
@@ -89,12 +89,14 @@ class Python24Parser(Python25Parser):
         while1stmt    ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM
         while1stmt    ::= SETUP_LOOP returns COME_FROM
         whilestmt     ::= SETUP_LOOP testexpr returns POP_BLOCK COME_FROM
+        with          ::= expr setupwith SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                          LOAD_CONST COME_FROM with_cleanup
+        with_as       ::= expr setupwithas store suite_stmts_opt POP_BLOCK
+                          LOAD_CONST COME_FROM with_cleanup
         with_cleanup  ::= LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
         with_cleanup  ::= LOAD_NAME DELETE_NAME WITH_CLEANUP END_FINALLY
-        withasstmt    ::= expr setupwithas store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM with_cleanup
-        with          ::= expr setupwith SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM with_cleanup
-        stmt ::= with
-        stmt ::= withasstmt
+        stmt          ::= with
+        stmt          ::= with_as
         """
         )
         super(Python24Parser, self).customize_grammar_rules(tokens, customize)
@@ -115,8 +117,8 @@ class Python24Parser(Python25Parser):
 
         lhs = rule[0]
         if lhs == "nop_stmt":
-            l = len(tokens)
-            if 0 <= l < len(tokens):
+            token_len = len(tokens)
+            if 0 <= token_len < len(tokens):
                 return not int(tokens[first].pattr) == tokens[last].offset
         elif lhs == "try_except":
             if last == len(tokens):

uncompyle6/parsers/parse25.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2018, 2020, 2022 Rocky Bernstein
+#  Copyright (c) 2016-2018, 2020, 2022, 2024 Rocky Bernstein
 """
 spark grammar differences over Python2.6 for Python 2.5.
 """
@@ -33,7 +33,7 @@ class Python25Parser(Python26Parser):
                       POP_BLOCK LOAD_CONST COME_FROM with_cleanup
 
         # Semantic actions want store to be at index 2
-        withasstmt ::= expr setupwithas store suite_stmts_opt
+        with_as ::= expr setupwithas store suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM with_cleanup
 
 
@@ -48,7 +48,7 @@ class Python25Parser(Python26Parser):
 
         # Python 2.6 omits the LOAD_FAST DELETE_FAST below
         # withas is allowed as a "from future" in 2.5
-        withasstmt ::= expr setupwithas store suite_stmts_opt
+        with_as ::= expr setupwithas store suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM
                        with_cleanup
 
@@ -67,7 +67,7 @@ class Python25Parser(Python26Parser):
         setupwith  ::= DUP_TOP LOAD_ATTR ROT_TWO LOAD_ATTR CALL_FUNCTION_0 POP_TOP
         with       ::= expr setupwith SETUP_FINALLY suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
-        withasstmt ::= expr setupwithas store suite_stmts_opt
+        with_as    ::= expr setupwithas store suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
         assert2       ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
         classdefdeco  ::= classdefdeco1 store

uncompyle6/parsers/parse26.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2022 Rocky Bernstein
+#  Copyright (c) 2017-2024 Rocky Bernstein
 """
 spark grammar differences over Python2 for Python 2.6.
 """
@@ -136,7 +136,7 @@ class Python26Parser(Python2Parser):
                         POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
 
         # Semantic actions want store to be at index 2
-        withasstmt ::= expr setupwithas store suite_stmts_opt
+        with_as ::= expr setupwithas store suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM WITH_CLEANUP END_FINALLY
 
         # This is truly weird. 2.7 does this (not including POP_TOP) with
@@ -307,21 +307,22 @@ class Python26Parser(Python2Parser):
 
         and                ::= expr JUMP_IF_FALSE POP_TOP expr JUMP_IF_FALSE POP_TOP
 
-        # compare_chained is like x <= y <= z
-        compare_chained    ::= expr compare_chained1 ROT_TWO COME_FROM POP_TOP _come_froms
-        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                               jmp_false compare_chained1 _come_froms
-        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                               jmp_false compare_chained2 _come_froms
+        # A "compare_chained" is two comparisons like x <= y <= z
+        compare_chained          ::= expr compared_chained_middle ROT_TWO
+                                     COME_FROM POP_TOP _come_froms
+        compared_chained_middle  ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                     jmp_false compared_chained_middle _come_froms
+        compared_chained_middle   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                     jmp_false compare_chained_right _come_froms
 
-        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                               jmp_false_then compare_chained1 _come_froms
-        compare_chained1   ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                               jmp_false_then compare_chained2 _come_froms
+        compared_chained_middle   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                      jmp_false_then compared_chained_middle _come_froms
+        compared_chained_middle   ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                      jmp_false_then compare_chained_right _come_froms
 
-        compare_chained2   ::= expr COMPARE_OP return_expr_lambda
-        compare_chained2   ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
-        compare_chained2   ::= expr COMPARE_OP RETURN_END_IF COME_FROM
+        compare_chained_right   ::= expr COMPARE_OP return_expr_lambda
+        compare_chained_right   ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
+        compare_chained_right   ::= expr COMPARE_OP RETURN_END_IF COME_FROM
 
         return_if_lambda   ::= RETURN_END_IF_LAMBDA POP_TOP
         stmt               ::= if_exp_lambda
@@ -351,9 +352,9 @@ class Python26Parser(Python2Parser):
     def customize_grammar_rules(self, tokens, customize):
         self.remove_rules(
             """
-        withasstmt ::= expr SETUP_WITH store suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM_WITH
-                WITH_CLEANUP END_FINALLY
+        with_as ::= expr SETUP_WITH store suite_stmts_opt
+                    POP_BLOCK LOAD_CONST COME_FROM_WITH
+                    WITH_CLEANUP END_FINALLY
         """
         )
         super(Python26Parser, self).customize_grammar_rules(tokens, customize)
@@ -390,7 +391,6 @@ class Python26Parser(Python2Parser):
             ("and", ("expr", "jmp_false", "expr", "come_from_opt")),
             ("assert_expr_and", ("assert_expr", "jmp_false", "expr")),
         ):
-
             # FIXME: workaround profiling bug
             if ast[1] is None:
                 return False
@@ -465,7 +465,7 @@ class Python26Parser(Python2Parser):
             ja_attr = ast[4].attr
             return tokens[last].offset != ja_attr
         elif lhs == "try_except":
-            # We need to distingush "try_except" from "tryelsestmt"; we do that
+            # We need to distinguish "try_except" from "tryelsestmt"; we do that
             # by checking the jump before the END_FINALLY
             # If we have:
             #    insn
@@ -490,7 +490,6 @@ class Python26Parser(Python2Parser):
                     ("JUMP_FORWARD", "RETURN_VALUE")
                 ) or (tokens[last - 3] == "JUMP_FORWARD" and tokens[last - 3].attr != 2)
         elif lhs == "tryelsestmt":
-
             # We need to distinguish "try_except" from "tryelsestmt"; we do that
             # by making sure that the jump before the except handler jumps to
             # code somewhere before the end of the construct.
@@ -565,7 +564,7 @@ if __name__ == "__main__":
         remain_tokens = set(tokens) - opcode_set
         import re
 
-        remain_tokens = set([re.sub("_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
         remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)

uncompyle6/parsers/parse27.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2020, 2022 Rocky Bernstein
+#  Copyright (c) 2016-2020, 2023 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <hartmut@goebel.noris.de>
 
@@ -37,11 +37,12 @@ class Python27Parser(Python2Parser):
         dict_comp ::= LOAD_DICTCOMP MAKE_FUNCTION_0 expr GET_ITER CALL_FUNCTION_1
 
         stmt           ::= dict_comp_func
+
         dict_comp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER store
-                           comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
+                           comp_iter JUMP_BACK ending_return
 
         set_comp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER store comp_iter
-                          JUMP_BACK RETURN_VALUE RETURN_LAST
+                          JUMP_BACK ending_return
 
         comp_iter     ::= comp_if_not
         comp_if_not   ::= expr jmp_true comp_iter
@@ -115,17 +116,18 @@ class Python27Parser(Python2Parser):
         or         ::= expr_jitop expr COME_FROM
         and        ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
 
-        # compare_chained{1,2} is used exclusively in chained_compare
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained1 COME_FROM
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained2 COME_FROM
+        # compare_chained{middle,2} is used exclusively in chained_compare
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                    JUMP_IF_FALSE_OR_POP compared_chained_middle
+                                    COME_FROM
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                    JUMP_IF_FALSE_OR_POP compare_chained_right COME_FROM
 
         return_lambda      ::= RETURN_VALUE
         return_lambda      ::= RETURN_VALUE_LAMBDA
 
-        compare_chained2 ::= expr COMPARE_OP return_lambda
-        compare_chained2 ::= expr COMPARE_OP return_lambda
+        compare_chained_right ::= expr COMPARE_OP return_lambda
+        compare_chained_right ::= expr COMPARE_OP return_lambda
 
         # if_exp_true are for conditions which always evaluate true
         # There is dead or non-optional remnants of the condition code though,
@@ -159,9 +161,9 @@ class Python27Parser(Python2Parser):
                        POP_BLOCK LOAD_CONST COME_FROM_WITH
                        WITH_CLEANUP END_FINALLY
 
-        withasstmt ::= expr SETUP_WITH store suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM_WITH
-                WITH_CLEANUP END_FINALLY
+        with_as    ::= expr SETUP_WITH store suite_stmts_opt
+                       POP_BLOCK LOAD_CONST COME_FROM_WITH
+                       WITH_CLEANUP END_FINALLY
 
         whilestmt         ::= SETUP_LOOP testexpr returns
                               _come_froms POP_BLOCK COME_FROM
@@ -177,11 +179,13 @@ class Python27Parser(Python2Parser):
         while1stmt        ::= SETUP_LOOP returns pb_come_from
         while1stmt        ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
 
-        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK _come_froms
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+                              _come_froms
 
         # Should this be JUMP_BACK+ ?
         # JUMP_BACK should all be to the same location
-        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK JUMP_BACK POP_BLOCK _come_froms
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK
+                              JUMP_BACK POP_BLOCK _come_froms
 
         while1elsestmt    ::= SETUP_LOOP l_stmts JUMP_BACK POP_BLOCK
                               else_suitel COME_FROM

uncompyle6/parsers/parse3.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2022 Rocky Bernstein
+#  Copyright (c) 2015-2024 Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -27,22 +27,24 @@ that a later phase can turn into a sequence of ASCII text.
 """
 
 import re
-from uncompyle6.scanners.tok import Token
+
+from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
 from uncompyle6.parser import PythonParser, PythonParserSingle, nop_func
 from uncompyle6.parsers.reducecheck import (
     and_invalid,
     except_handler_else,
     ifelsestmt,
-    ifstmt,
     iflaststmt,
+    ifstmt,
     or_check,
     testtrue,
     tryelsestmtl3,
     tryexcept,
-    while1stmt
+    while1stmt,
 )
 from uncompyle6.parsers.treenode import SyntaxTree
-from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+from uncompyle6.scanners.tok import Token
 
 
 class Python3Parser(PythonParser):
@@ -79,11 +81,13 @@ class Python3Parser(PythonParser):
 
         stmt ::= set_comp_func
 
+        # TODO this can be simplified
         set_comp_func ::= BUILD_SET_0 LOAD_ARG FOR_ITER store comp_iter
-                          JUMP_BACK RETURN_VALUE RETURN_LAST
-
+                          JUMP_BACK ending_return
+        set_comp_func ::= BUILD_SET_0 LOAD_FAST FOR_ITER store comp_iter
+                          JUMP_BACK ending_return
         set_comp_func ::= BUILD_SET_0 LOAD_ARG FOR_ITER store comp_iter
-                          COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST
+                          COME_FROM JUMP_BACK ending_return
 
         comp_body ::= dict_comp_body
         comp_body ::= set_comp_body
@@ -96,11 +100,17 @@ class Python3Parser(PythonParser):
         """
 
     def p_dict_comp3(self, args):
-        """"
+        """ "
         expr ::= dict_comp
         stmt ::= dict_comp_func
         dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store
                            comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
+        dict_comp_func ::= BUILD_MAP_0 LOAD_ARG FOR_ITER store
+                           comp_iter JUMP_BACK RETURN_VALUE_LAMBDA LAMBDA_MARKER
+        dict_comp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER store
+                           comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
+        dict_comp_func ::= BUILD_MAP_0 LOAD_FAST FOR_ITER store
+                           comp_iter JUMP_BACK RETURN_VALUE_LAMBDA LAMBDA_MARKER
 
         comp_iter     ::= comp_if_not
         comp_if_not   ::= expr jmp_true comp_iter
@@ -277,9 +287,9 @@ class Python3Parser(PythonParser):
                     POP_BLOCK LOAD_CONST COME_FROM_WITH
                     WITH_CLEANUP END_FINALLY
 
-        withasstmt ::= expr SETUP_WITH store suite_stmts_opt
-                POP_BLOCK LOAD_CONST COME_FROM_WITH
-                WITH_CLEANUP END_FINALLY
+        with_as ::= expr SETUP_WITH store suite_stmts_opt
+                    POP_BLOCK LOAD_CONST COME_FROM_WITH
+                    WITH_CLEANUP END_FINALLY
 
         expr_jt     ::= expr jmp_true
         expr_jitop  ::= expr JUMP_IF_TRUE_OR_POP
@@ -346,14 +356,15 @@ class Python3Parser(PythonParser):
         # FIXME: Common with 2.7
         ret_and    ::= expr JUMP_IF_FALSE_OR_POP return_expr_or_cond COME_FROM
         ret_or     ::= expr JUMP_IF_TRUE_OR_POP return_expr_or_cond COME_FROM
-        if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM return_expr_or_cond
+        if_exp_ret ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF COME_FROM
+                       return_expr_or_cond
 
 
-        # compare_chained1 is used exclusively in chained_compare
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained1 COME_FROM
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained2 COME_FROM
+        # compared_chained_middle is used exclusively in chained_compare
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                                    compared_chained_middle COME_FROM
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                                    compare_chained_right COME_FROM
         """
 
     def p_stmt3(self, args):
@@ -419,24 +430,24 @@ class Python3Parser(PythonParser):
         for               ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
                               COME_FROM_LOOP
 
-        forelsestmt       ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suite
-                              COME_FROM_LOOP
+        forelsestmt       ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
+                              else_suite COME_FROM_LOOP
 
-        forelselaststmt   ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitec
-                              COME_FROM_LOOP
+        forelselaststmt   ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
+                              else_suitec COME_FROM_LOOP
 
-        forelselaststmtl  ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK else_suitel
-                              COME_FROM_LOOP
+        forelselaststmtl  ::= SETUP_LOOP expr for_iter store for_block POP_BLOCK
+                              else_suitel COME_FROM_LOOP
 
-        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
-                              COME_FROM_LOOP
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt COME_FROM JUMP_BACK
+                              POP_BLOCK COME_FROM_LOOP
 
-        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK
-                              COME_FROM_LOOP
+        whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+                              JUMP_BACK COME_FROM_LOOP
         whilestmt         ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
                               COME_FROM_LOOP
 
-        whilestmt         ::= SETUP_LOOP testexpr returns          POP_BLOCK
+        whilestmt         ::= SETUP_LOOP testexpr returns               POP_BLOCK
                               COME_FROM_LOOP
 
         while1elsestmt    ::= SETUP_LOOP          l_stmts     JUMP_BACK
@@ -510,7 +521,7 @@ class Python3Parser(PythonParser):
                         expr
                         call
                         CALL_FUNCTION_3
-         """
+        """
         # FIXME: I bet this can be simplified
         # look for next MAKE_FUNCTION
         for i in range(i + 1, len(tokens)):
@@ -616,7 +627,11 @@ class Python3Parser(PythonParser):
         self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
         if "LOAD_BUILD_CLASS" in self.seen_ops:
-            if next_token == "CALL_FUNCTION" and next_token.attr == 1 and pos_args_count > 1:
+            if (
+                next_token == "CALL_FUNCTION"
+                and next_token.attr == 1
+                and pos_args_count > 1
+            ):
                 rule = "classdefdeco2 ::= LOAD_BUILD_CLASS mkfunc %s%s_%d" % (
                     ("expr " * (pos_args_count - 1)),
                     opname,
@@ -625,7 +640,7 @@ class Python3Parser(PythonParser):
                 self.add_unique_rule(rule, token.kind, uniq_param, customize)
 
     def add_make_function_rule(self, rule, opname, attr, customize):
-        """Python 3.3 added a an addtional LOAD_STR before MAKE_FUNCTION and
+        """Python 3.3 added a an additional LOAD_STR before MAKE_FUNCTION and
         this has an effect on many rules.
         """
         if self.version >= (3, 3):
@@ -755,18 +770,24 @@ class Python3Parser(PythonParser):
 
             elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
                 if opname == "BUILD_CONST_DICT":
-                    rule = """
+                    rule = (
+                        """
                            add_consts          ::= ADD_VALUE*
                            const_list          ::= COLLECTION_START add_consts %s
                            dict                ::= const_list
                            expr                ::= dict
-                           """ % opname
+                           """
+                        % opname
+                    )
                 else:
-                    rule = """
+                    rule = (
+                        """
                            add_consts          ::= ADD_VALUE*
                            const_list          ::= COLLECTION_START add_consts %s
                            expr                ::= const_list
-                           """ % opname
+                           """
+                        % opname
+                    )
                 self.addRule(rule, nop_func)
 
             elif opname.startswith("BUILD_DICT_OLDER"):
@@ -845,18 +866,24 @@ class Python3Parser(PythonParser):
 
             elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
                 if opname == "BUILD_CONST_DICT":
-                    rule = """
+                    rule = (
+                        """
                            add_consts          ::= ADD_VALUE*
                            const_list          ::= COLLECTION_START add_consts %s
                            dict                ::= const_list
                            expr                ::= dict
-                           """ % opname
+                           """
+                        % opname
+                    )
                 else:
-                    rule = """
+                    rule = (
+                        """
                            add_consts          ::= ADD_VALUE*
                            const_list          ::= COLLECTION_START add_consts %s
                            expr                ::= const_list
-                           """ % opname
+                           """
+                        % opname
+                    )
                 self.addRule(rule, nop_func)
 
             elif opname_base in (
@@ -937,7 +964,6 @@ class Python3Parser(PythonParser):
                     "CALL_FUNCTION_VAR_KW",
                 )
             ) or opname.startswith("CALL_FUNCTION_KW"):
-
                 if opname == "CALL_FUNCTION" and token.attr == 1:
                     rule = """
                      dict_comp    ::= LOAD_DICTCOMP LOAD_STR MAKE_FUNCTION_0 expr
@@ -1059,7 +1085,9 @@ class Python3Parser(PythonParser):
                 )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_LISTCOMP":
-                self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_SETCOMP":
                 # Should this be generalized and put under MAKE_FUNCTION?
@@ -1113,7 +1141,8 @@ class Python3Parser(PythonParser):
                 if has_get_iter_call_function1:
                     rule_pat = (
                         "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
-                        "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
+                        "GET_ITER CALL_FUNCTION_1"
+                        % ("pos_arg " * pos_args_count, opname)
                     )
                     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
@@ -1127,7 +1156,7 @@ class Python3Parser(PythonParser):
                             #   and have GET_ITER CALL_FUNCTION_1
                             # Todo: For Pypy we need to modify this slightly
                             rule_pat = (
-                                "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
+                                "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
                                 % ("pos_arg " * pos_args_count, opname)
                             )
@@ -1181,6 +1210,8 @@ class Python3Parser(PythonParser):
                     self.add_unique_rule(rule, opname, token.attr, customize)
 
                 elif (3, 3) <= self.version < (3, 6):
+                    # FIXME move this into version-specific custom rules.
+                    # In fact, some of this has been done for 3.3.
                     if annotate_args > 0:
                         rule = (
                             "mkfunc_annotate ::= %s%s%sannotate_tuple load_closure LOAD_CODE LOAD_STR %s"
@@ -1192,14 +1223,28 @@ class Python3Parser(PythonParser):
                             )
                         )
                     else:
-                        rule = "mkfunc ::= %s%sload_closure LOAD_CODE LOAD_STR %s" % (
-                            kwargs_str,
-                            "pos_arg " * pos_args_count,
+                        if self.version == (3, 3):
+                            # 3.3 puts kwargs before pos_arg
+                            pos_kw_tuple = (
+                                ("kwargs " * kw_args_count),
+                                ("pos_arg " * pos_args_count),
+                            )
+                        else:
+                            # 3.4 and 3.5 puts pos_arg before kwargs
+                            pos_kw_tuple = (
+                                "pos_arg " * (pos_args_count),
+                                ("kwargs " * kw_args_count),
+                            )
+                        rule = (
+                            "mkfunc ::= %s%s%s " "load_closure LOAD_CODE LOAD_STR %s"
+                        ) % (
+                            pos_kw_tuple[0],
+                            pos_kw_tuple[1],
+                            "annotate_tuple " * (annotate_args),
                             opname,
                         )
                     self.add_unique_rule(rule, opname, token.attr, customize)
 
-
                 if self.version >= (3, 4):
                     if not self.is_pypy:
                         load_op = "LOAD_STR"
@@ -1283,14 +1328,16 @@ class Python3Parser(PythonParser):
                     if has_get_iter_call_function1:
                         rule_pat = (
                             "generator_exp ::= %sload_genexpr %%s%s expr "
-                            "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
+                            "GET_ITER CALL_FUNCTION_1"
+                            % ("pos_arg " * pos_args_count, opname)
                         )
                         self.add_make_function_rule(
                             rule_pat, opname, token.attr, customize
                         )
                         rule_pat = (
                             "generator_exp ::= %sload_closure load_genexpr %%s%s expr "
-                            "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
+                            "GET_ITER CALL_FUNCTION_1"
+                            % ("pos_arg " * pos_args_count, opname)
                         )
                         self.add_make_function_rule(
                             rule_pat, opname, token.attr, customize
@@ -1303,14 +1350,14 @@ class Python3Parser(PythonParser):
                                 # 'exprs' in the rule above into a
                                 # tuple.
                                 rule_pat = (
-                                    "listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
+                                    "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
                                     "expr GET_ITER CALL_FUNCTION_1" % (opname,)
                                 )
                                 self.add_make_function_rule(
                                     rule_pat, opname, token.attr, customize
                                 )
                             rule_pat = (
-                                "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                                "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
                                 % ("expr " * pos_args_count, opname)
                             )
@@ -1342,7 +1389,8 @@ class Python3Parser(PythonParser):
                 if has_get_iter_call_function1:
                     rule_pat = (
                         "generator_exp ::= %sload_genexpr %%s%s expr "
-                        "GET_ITER CALL_FUNCTION_1" % ("pos_arg " * pos_args_count, opname)
+                        "GET_ITER CALL_FUNCTION_1"
+                        % ("pos_arg " * pos_args_count, opname)
                     )
                     self.add_make_function_rule(rule_pat, opname, token.attr, customize)
 
@@ -1353,8 +1401,9 @@ class Python3Parser(PythonParser):
                         #   and have GET_ITER CALL_FUNCTION_1
                         # Todo: For Pypy we need to modify this slightly
                         rule_pat = (
-                            "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
-                            "GET_ITER CALL_FUNCTION_1" % ("expr " * pos_args_count, opname)
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "GET_ITER CALL_FUNCTION_1"
+                            % ("expr " * pos_args_count, opname)
                         )
                         self.add_make_function_rule(
                             rule_pat, opname, token.attr, customize
@@ -1433,17 +1482,14 @@ class Python3Parser(PythonParser):
                         )
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = (
-                            "mkfunc_annotate ::= %s%sannotate_tuple LOAD_CODE LOAD_STR %s"
-                            % (
-                                ("pos_arg " * pos_args_count),
-                                ("annotate_arg " * annotate_args),
-                                opname,
-                            )
+                            "mkfunc_annotate ::= %s%sannotate_tuple LOAD_CODE "
+                            "LOAD_STR %s"
+                        ) % (
+                            ("pos_arg " * pos_args_count),
+                            ("annotate_arg " * annotate_args),
+                            opname,
                         )
                     if self.version >= (3, 3):
-                        # Normally we remove EXTENDED_ARG from the opcodes, but in the case of
-                        # annotated functions can use the EXTENDED_ARG tuple to signal we have an annotated function.
-                        # Yes this is a little hacky
                         if self.version == (3, 3):
                             # 3.3 puts kwargs before pos_arg
                             pos_kw_tuple = (
@@ -1451,34 +1497,23 @@ class Python3Parser(PythonParser):
                                 ("pos_arg " * pos_args_count),
                             )
                         else:
-                            # 3.4 and 3.5puts pos_arg before kwargs
+                            # 3.4 and 3.5 puts pos_arg before kwargs
                             pos_kw_tuple = (
                                 "pos_arg " * (pos_args_count),
                                 ("kwargs " * kw_args_count),
                             )
                         rule = (
-                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s"
-                            % (
-                                pos_kw_tuple[0],
-                                pos_kw_tuple[1],
-                                ("call " * annotate_args),
-                                opname,
-                            )
-                        )
-                        self.add_unique_rule(rule, opname, token.attr, customize)
-                        rule = (
-                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE LOAD_STR EXTENDED_ARG %s"
-                            % (
-                                pos_kw_tuple[0],
-                                pos_kw_tuple[1],
-                                ("annotate_arg " * annotate_args),
-                                opname,
-                            )
+                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE "
+                            "LOAD_STR %s"
+                        ) % (
+                            pos_kw_tuple[0],
+                            pos_kw_tuple[1],
+                            ("annotate_arg " * annotate_args),
+                            opname,
                         )
                     else:
-                        # See above comment about use of EXTENDED_ARG
                         rule = (
-                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s"
+                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
                             % (
                                 ("kwargs " * kw_args_count),
                                 ("pos_arg " * (pos_args_count)),
@@ -1488,7 +1523,7 @@ class Python3Parser(PythonParser):
                         )
                         self.add_unique_rule(rule, opname, token.attr, customize)
                         rule = (
-                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE EXTENDED_ARG %s"
+                            "mkfunc_annotate ::= %s%s%sannotate_tuple LOAD_CODE %s"
                             % (
                                 ("kwargs " * kw_args_count),
                                 ("pos_arg " * pos_args_count),
@@ -1585,7 +1620,7 @@ class Python3Parser(PythonParser):
         }
 
         if self.version == (3, 6):
-            self.reduce_check_table["and"] =  and_invalid
+            self.reduce_check_table["and"] = and_invalid
             self.check_reduce["and"] = "AST"
 
         self.check_reduce["annotate_tuple"] = "noAST"
@@ -1615,7 +1650,7 @@ class Python3Parser(PythonParser):
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
         lhs = rule[0]
         n = len(tokens)
-        last = min(last, n-1)
+        last = min(last, n - 1)
         fn = self.reduce_check_table.get(lhs, None)
         if fn:
             if fn(self, lhs, n, rule, ast, tokens, first, last):
@@ -1641,13 +1676,18 @@ class Python3Parser(PythonParser):
                 condition_jump2 = tokens[min(last - 1, len(tokens) - 1)]
                 # If there are two *distinct* condition jumps, they should not jump to the
                 # same place. Otherwise we have some sort of "and"/"or".
-                if condition_jump2.kind.startswith("POP_JUMP_IF") and condition_jump != condition_jump2:
+                if (
+                    condition_jump2.kind.startswith("POP_JUMP_IF")
+                    and condition_jump != condition_jump2
+                ):
                     return condition_jump.attr == condition_jump2.attr
 
-                if tokens[last] == "COME_FROM" and tokens[last].off2int() != condition_jump.attr:
+                if (
+                    tokens[last] == "COME_FROM"
+                    and tokens[last].off2int() != condition_jump.attr
+                ):
                     return False
 
-
                 # if condition_jump.attr < condition_jump2.off2int():
                 #     print("XXX", first, last)
                 #     for t in range(first, last): print(tokens[t])
@@ -1669,7 +1709,6 @@ class Python3Parser(PythonParser):
                 < tokens[last].off2int()
             )
         elif lhs == "while1stmt":
-
             if while1stmt(self, lhs, n, rule, ast, tokens, first, last):
                 return True
 
@@ -1691,7 +1730,6 @@ class Python3Parser(PythonParser):
                     return True
             return False
         elif lhs == "while1elsestmt":
-
             n = len(tokens)
             if last == n:
                 # Adjust for fuzziness in parsing

uncompyle6/parsers/parse30.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2017, 2022 Rocky Bernstein
+#  Copyright (c) 2016-2017, 2022-2024 Rocky Bernstein
 """
 spark grammar differences over Python 3.1 for Python 3.0.
 """
@@ -7,20 +7,21 @@ from __future__ import print_function
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse31 import Python31Parser
 
-class Python30Parser(Python31Parser):
 
+class Python30Parser(Python31Parser):
     def p_30(self, args):
         """
 
         pt_bp             ::= POP_TOP POP_BLOCK
 
-        assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1 COME_FROM POP_TOP
-        assert2           ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1 RAISE_VARARGS_1
-                              come_froms
+        assert            ::= assert_expr jmp_true LOAD_ASSERT RAISE_VARARGS_1
+                              COME_FROM POP_TOP
+        assert2           ::= assert_expr jmp_true LOAD_ASSERT expr CALL_FUNCTION_1
+                              RAISE_VARARGS_1 come_froms
         call_stmt         ::= expr _come_froms POP_TOP
 
-        return_if_lambda  ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP
-        compare_chained2  ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
+        return_if_lambda       ::= RETURN_END_IF_LAMBDA COME_FROM POP_TOP
+        compare_chained_right  ::= expr COMPARE_OP RETURN_END_IF_LAMBDA
 
         # FIXME: combine with parse3.2
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt
@@ -30,8 +31,8 @@ class Python30Parser(Python31Parser):
 
         # In many ways Python 3.0 code generation is more like Python 2.6 than
         # it is 2.7 or 3.1. So we have a number of 2.6ish (and before) rules below
-        # Specifically POP_TOP is more prevelant since there is no POP_JUMP_IF_...
-        # instructions
+        # Specifically POP_TOP is more prevalant since there is no POP_JUMP_IF_...
+        # instructions.
 
         _ifstmts_jump  ::= c_stmts JUMP_FORWARD _come_froms POP_TOP COME_FROM
         _ifstmts_jump  ::= c_stmts COME_FROM POP_TOP
@@ -65,7 +66,7 @@ class Python30Parser(Python31Parser):
         iflaststmt  ::= testexpr c_stmts_opt JUMP_ABSOLUTE COME_FROM POP_TOP
 
 
-        withasstmt    ::= expr setupwithas store suite_stmts_opt
+        with_as    ::= expr setupwithas store suite_stmts_opt
                           POP_BLOCK LOAD_CONST COME_FROM_FINALLY
                           LOAD_FAST DELETE_FAST WITH_CLEANUP END_FINALLY
         setupwithas   ::= DUP_TOP LOAD_ATTR STORE_FAST LOAD_ATTR CALL_FUNCTION_0 setup_finally
@@ -73,9 +74,10 @@ class Python30Parser(Python31Parser):
 
         # Need to keep LOAD_FAST as index 1
         set_comp_header  ::= BUILD_SET_0 DUP_TOP STORE_FAST
+
         set_comp_func ::= set_comp_header
                           LOAD_ARG FOR_ITER store comp_iter
-                          JUMP_BACK COME_FROM POP_TOP JUMP_BACK RETURN_VALUE RETURN_LAST
+                          JUMP_BACK ending_return
 
         list_comp_header ::= BUILD_LIST_0 DUP_TOP STORE_FAST
         list_comp        ::= list_comp_header
@@ -105,7 +107,7 @@ class Python30Parser(Python31Parser):
         dict_comp_func   ::= BUILD_MAP_0
                              DUP_TOP STORE_FAST
                              LOAD_ARG FOR_ITER store
-                             dict_comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
+                             dict_comp_iter JUMP_BACK ending_return
 
         stmt         ::= try_except30
         try_except30 ::= SETUP_EXCEPT suite_stmts_opt
@@ -115,7 +117,7 @@ class Python30Parser(Python31Parser):
         # From Python 2.6
 
 
-	lc_body     ::= LOAD_FAST expr LIST_APPEND
+        lc_body     ::= LOAD_FAST expr LIST_APPEND
         lc_body     ::= LOAD_NAME expr LIST_APPEND
         list_if     ::= expr jmp_false_then list_iter
         list_if_not ::= expr jmp_true list_iter JUMP_BACK come_froms POP_TOP
@@ -205,27 +207,32 @@ class Python30Parser(Python31Parser):
                            come_froms POP_TOP POP_BLOCK COME_FROM_LOOP
 
 
-        # compare_chained is like x <= y <= z
-        compare_chained1  ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                              jmp_false compare_chained1 _come_froms
-        compare_chained1  ::= expr DUP_TOP ROT_THREE COMPARE_OP
-                              jmp_false compare_chained2 _come_froms
-        compare_chained2 ::= expr COMPARE_OP RETURN_END_IF
+        # A "compare_chained" is two comparisons like x <= y <= z
+        compared_chained_middle  ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                     jmp_false compared_chained_middle _come_froms
+        compared_chained_middle  ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                     jmp_false compare_chained_right _come_froms
+        compare_chained_right ::= expr COMPARE_OP RETURN_END_IF
         """
 
-
     def remove_rules_30(self):
-        self.remove_rules("""
+        self.remove_rules(
+            """
 
         # The were found using grammar coverage
         while1stmt     ::= SETUP_LOOP l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
         whileTruestmt  ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
-        whileelsestmt  ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK else_suitel COME_FROM_LOOP
-        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM_LOOP
-        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK COME_FROM_LOOP
+        whileelsestmt  ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+                           else_suitel COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+                           COME_FROM_LOOP
+        whilestmt      ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK JUMP_BACK
+                           COME_FROM_LOOP
         whilestmt      ::= SETUP_LOOP testexpr returns POP_TOP POP_BLOCK COME_FROM_LOOP
-        withasstmt     ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
-        with           ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST COME_FROM_WITH WITH_CLEANUP END_FINALLY
+        with_as        ::= expr SETUP_WITH store suite_stmts_opt POP_BLOCK LOAD_CONST
+                           COME_FROM_WITH WITH_CLEANUP END_FINALLY
+        with           ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK LOAD_CONST
+                           COME_FROM_WITH WITH_CLEANUP END_FINALLY
 
         # lc_body ::= LOAD_FAST expr LIST_APPEND
         # lc_body ::= LOAD_NAME expr LIST_APPEND
@@ -270,10 +277,11 @@ class Python30Parser(Python31Parser):
         jmp_true         ::= JUMP_IF_TRUE_OR_POP POP_TOP
         jmp_true         ::= POP_JUMP_IF_TRUE
 
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained1 COME_FROM
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained2 COME_FROM
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                    JUMP_IF_FALSE_OR_POP compared_chained_middle
+                                    COME_FROM
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP
+                                    JUMP_IF_FALSE_OR_POP compare_chained_right COME_FROM
         ret_or           ::= expr JUMP_IF_TRUE_OR_POP  return_expr_or_cond COME_FROM
         ret_and          ::= expr JUMP_IF_FALSE_OR_POP return_expr_or_cond COME_FROM
         if_exp_ret       ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF
@@ -282,29 +290,30 @@ class Python30Parser(Python31Parser):
         or               ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and              ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
         and              ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
-        """)
+        """
+        )
 
     def customize_grammar_rules(self, tokens, customize):
         super(Python30Parser, self).customize_grammar_rules(tokens, customize)
         self.remove_rules_30()
 
         self.check_reduce["iflaststmtl"] = "AST"
-        self.check_reduce['ifstmt'] = "AST"
+        self.check_reduce["ifstmt"] = "AST"
         self.check_reduce["ifelsestmtc"] = "AST"
         self.check_reduce["ifelsestmt"] = "AST"
         # self.check_reduce["and"] = "stmt"
         return
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
-        invalid = super(Python30Parser,
-                        self).reduce_is_invalid(rule, ast,
-                                                tokens, first, last)
+        invalid = super(Python30Parser, self).reduce_is_invalid(
+            rule, ast, tokens, first, last
+        )
         if invalid:
             return invalid
         lhs = rule[0]
         if (
-            lhs in ("iflaststmtl", "ifstmt",
-                        "ifelsestmt", "ifelsestmtc") and ast[0] == "testexpr"
+            lhs in ("iflaststmtl", "ifstmt", "ifelsestmt", "ifelsestmtc")
+            and ast[0] == "testexpr"
         ):
             testexpr = ast[0]
             if testexpr[0] == "testfalse":
@@ -312,7 +321,10 @@ class Python30Parser(Python31Parser):
                 if lhs == "ifelsestmtc" and ast[2] == "jump_absolute_else":
                     jump_absolute_else = ast[2]
                     come_from = jump_absolute_else[2]
-                    return come_from == "COME_FROM" and come_from.attr < tokens[first].offset
+                    return (
+                        come_from == "COME_FROM"
+                        and come_from.attr < tokens[first].offset
+                    )
                     pass
                 elif lhs in ("ifelsestmt", "ifelsestmtc") and ast[2] == "jump_cf_pop":
                     jump_cf_pop = ast[2]
@@ -335,11 +347,11 @@ class Python30Parser(Python31Parser):
                     jmp_false = testfalse[1]
                     if last == len(tokens):
                         last -= 1
-                    while (isinstance(tokens[first].offset, str) and first < last):
+                    while isinstance(tokens[first].offset, str) and first < last:
                         first += 1
                     if first == last:
                         return True
-                    while (first < last and isinstance(tokens[last].offset, str)):
+                    while first < last and isinstance(tokens[last].offset, str):
                         last -= 1
                     if rule[0] == "iflaststmtl":
                         return not (jmp_false[0].attr <= tokens[last].offset)
@@ -347,8 +359,9 @@ class Python30Parser(Python31Parser):
                         jmp_false_target = jmp_false[0].attr
                         if tokens[first].offset > jmp_false_target:
                             return True
-                        return (
-                            (jmp_false_target > tokens[last].offset) and tokens[last] != "JUMP_FORWARD")
+                        return (jmp_false_target > tokens[last].offset) and tokens[
+                            last
+                        ] != "JUMP_FORWARD"
                     pass
                 pass
             pass
@@ -357,33 +370,43 @@ class Python30Parser(Python31Parser):
 
     pass
 
+
 class Python30ParserSingle(Python30Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Check grammar
     p = Python30Parser()
     p.remove_rules_30()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
+
     if PYTHON_VERSION_TRIPLE[:2] == (3, 0):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+        opcode_set = set(s.opc.opname).union(
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split()
+            )
+        )
         ## FIXME: try this
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '',  t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         import sys
+
         if len(sys.argv) > 1:
             from spark_parser.spark import rule2str
+
             for rule in sorted(p.rule2name.items()):
                 print(rule2str(rule[0]))

uncompyle6/parsers/parse31.py

@@ -23,7 +23,7 @@ class Python31Parser(Python32Parser):
         # Keeps Python 3.1 "with .. as" designator in the same position as it is in other version.
         setupwithas31  ::= setupwithas SETUP_FINALLY load delete
 
-        withasstmt     ::= expr setupwithas31 store
+        with_as        ::= expr setupwithas31 store
                            suite_stmts_opt
                            POP_BLOCK LOAD_CONST COME_FROM_FINALLY
                            load delete WITH_CLEANUP END_FINALLY

uncompyle6/parsers/parse32.py

@@ -25,10 +25,9 @@ class Python32Parser(Python3Parser):
         """
         if_exp            ::= expr jmp_false expr jump_forward_else expr COME_FROM
 
-        # compare_chained2 is used in a "chained_compare": x <= y <= z
-        # used exclusively in compare_chained
-        compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
-        compare_chained2 ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
+        # compare_chained_right is used in a "chained_compare": x <= y <= z
+        compare_chained_right ::= expr COMPARE_OP RETURN_VALUE
+        compare_chained_right ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
 
         # Python < 3.5 no POP BLOCK
         whileTruestmt  ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM_LOOP

uncompyle6/parsers/parse33.py

@@ -1,31 +1,38 @@
-#  Copyright (c) 2016 Rocky Bernstein
+#  Copyright (c) 2016, 2024 Rocky Bernstein
 """
 spark grammar differences over Python 3.2 for Python 3.3.
 """
-from __future__ import print_function
 
 from uncompyle6.parser import PythonParserSingle
 from uncompyle6.parsers.parse32 import Python32Parser
 
 
 class Python33Parser(Python32Parser):
-
     def p_33on(self, args):
         """
         # Python 3.3+ adds yield from.
         expr          ::= yield_from
         yield_from    ::= expr expr YIELD_FROM
+        stmt         ::= genexpr_func
         """
 
     def customize_grammar_rules(self, tokens, customize):
-        self.remove_rules("""
+        self.remove_rules(
+            """
         # 3.3+ adds POP_BLOCKS
-        genexpr_func  ::= LOAD_ARG FOR_ITER store comp_iter JUMP_BACK
         whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK NOP COME_FROM_LOOP
         whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP COME_FROM_LOOP
-        """)
+        """
+        )
         super(Python33Parser, self).customize_grammar_rules(tokens, customize)
+
+        # FIXME: move 3.3 stuff out of parse3.py and put it here.
+        # for i, token in enumerate(tokens):
+        #     opname = token.kind
+        #     opname_base = opname[: opname.rfind("_")]
+
         return
 
+
 class Python33ParserSingle(Python33Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse34.py

@@ -52,6 +52,8 @@ class Python34Parser(Python33Parser):
         yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
 
         _ifstmts_jump ::= c_stmts_opt JUMP_ABSOLUTE JUMP_FORWARD COME_FROM
+
+        genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter JUMP_BACK
         """
 
     def customize_grammar_rules(self, tokens, customize):

uncompyle6/parsers/parse35.py

@@ -1,15 +1,17 @@
-#  Copyright (c) 2016-2017, 2019, 2021 Rocky Bernstein
+#  Copyright (c) 2016-2017, 2019, 2021, 2023-2024
+#  Rocky Bernstein
 """
 spark grammar differences over Python 3.4 for Python 3.5.
 """
 from __future__ import print_function
 
-from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse34 import Python34Parser
 
-class Python35Parser(Python34Parser):
 
+class Python35Parser(Python34Parser):
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         super(Python35Parser, self).__init__(debug_parser)
         self.customized = {}
@@ -55,7 +57,7 @@ class Python35Parser(Python34Parser):
                        POP_BLOCK LOAD_CONST COME_FROM_WITH
                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-        withasstmt ::= expr
+        with_as ::= expr
                        SETUP_WITH store suite_stmts_opt
                        POP_BLOCK LOAD_CONST COME_FROM_WITH
                        WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
@@ -114,7 +116,7 @@ class Python35Parser(Python34Parser):
         ifelsestmtl ::= testexpr c_stmts_opt jb_else else_suitel
 
         # 3.5 Has jump optimization which can route the end of an
-        # "if/then" back to to a loop just before an else.
+        # "if/then" back to a loop just before an else.
         jump_absolute_else ::= jb_else
         jump_absolute_else ::= CONTINUE ELSE
 
@@ -135,40 +137,42 @@ class Python35Parser(Python34Parser):
         """
 
     def customize_grammar_rules(self, tokens, customize):
-        self.remove_rules("""
+        self.remove_rules(
+            """
           yield_from ::= expr GET_ITER LOAD_CONST YIELD_FROM
           yield_from ::= expr expr YIELD_FROM
           with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt
                          POP_BLOCK LOAD_CONST COME_FROM_WITH
                          WITH_CLEANUP END_FINALLY
-          withasstmt ::= expr SETUP_WITH store suite_stmts_opt
+          with_as ::= expr SETUP_WITH store suite_stmts_opt
                          POP_BLOCK LOAD_CONST COME_FROM_WITH
                          WITH_CLEANUP END_FINALLY
-        """)
+        """
+        )
         super(Python35Parser, self).customize_grammar_rules(tokens, customize)
         for i, token in enumerate(tokens):
             opname = token.kind
-            if opname == 'LOAD_ASSERT':
-                if 'PyPy' in customize:
+            if opname == "LOAD_ASSERT":
+                if "PyPy" in customize:
                     rules_str = """
                     stmt ::= JUMP_IF_NOT_DEBUG stmts COME_FROM
                     """
                     self.add_unique_doc_rules(rules_str, customize)
             # FIXME: I suspect this is wrong for 3.6 and 3.5, but
             # I haven't verified what the 3.7ish fix is
-            elif opname == 'BUILD_MAP_UNPACK_WITH_CALL':
+            elif opname == "BUILD_MAP_UNPACK_WITH_CALL":
                 if self.version < (3, 7):
                     self.addRule("expr ::= unmapexpr", nop_func)
                     nargs = token.attr % 256
                     map_unpack_n = "map_unpack_%s" % nargs
-                    rule = map_unpack_n + ' ::= ' + 'expr ' * (nargs)
+                    rule = map_unpack_n + " ::= " + "expr " * (nargs)
                     self.addRule(rule, nop_func)
                     rule = "unmapexpr ::=  %s %s" % (map_unpack_n, opname)
                     self.addRule(rule, nop_func)
-                    call_token = tokens[i+1]
-                    rule = 'call ::= expr unmapexpr ' + call_token.kind
+                    call_token = tokens[i + 1]
+                    rule = "call ::= expr unmapexpr " + call_token.kind
                     self.addRule(rule, nop_func)
-            elif opname == 'BEFORE_ASYNC_WITH' and self.version < (3, 8):
+            elif opname == "BEFORE_ASYNC_WITH" and self.version < (3, 8):
                 # Some Python 3.5+ async additions
                 rules_str = """
                    stmt               ::= async_with_stmt
@@ -199,24 +203,27 @@ class Python35Parser(Python34Parser):
                                           async_with_post
                 """
                 self.addRule(rules_str, nop_func)
-            elif opname == 'BUILD_MAP_UNPACK':
-                self.addRule("""
+            elif opname == "BUILD_MAP_UNPACK":
+                self.addRule(
+                    """
                    expr        ::= dict_unpack
                    dict_unpack ::= dict_comp BUILD_MAP_UNPACK
-                   """, nop_func)
+                   """,
+                    nop_func,
+                )
 
-            elif opname == 'SETUP_WITH':
+            elif opname == "SETUP_WITH":
                 # Python 3.5+ has WITH_CLEANUP_START/FINISH
                 rules_str = """
-                  with ::= expr
-                           SETUP_WITH POP_TOP suite_stmts_opt
-                           POP_BLOCK LOAD_CONST COME_FROM_WITH
-                           WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                  with    ::= expr
+                              SETUP_WITH POP_TOP suite_stmts_opt
+                              POP_BLOCK LOAD_CONST COME_FROM_WITH
+                              WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
-                  withasstmt ::= expr
-                       SETUP_WITH store suite_stmts_opt
-                       POP_BLOCK LOAD_CONST COME_FROM_WITH
-                       WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
+                  with_as ::= expr
+                              SETUP_WITH store suite_stmts_opt
+                              POP_BLOCK LOAD_CONST COME_FROM_WITH
+                              WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
                 """
                 self.addRule(rules_str, nop_func)
             pass
@@ -230,19 +237,24 @@ class Python35Parser(Python34Parser):
         #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
         #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
         # Yes, this computation based on instruction name is a little bit hoaky.
-        nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
+        nak = (len(opname) - len("CALL_FUNCTION")) // 3
         uniq_param = args_kw + args_pos
 
-        if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops):
-            rule = ('async_call ::= expr ' +
-                    ('pos_arg ' * args_pos) +
-                    ('kwarg ' * args_kw) +
-                    'expr ' * nak + token.kind +
-                    ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
+        if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+            rule = (
+                "async_call ::= expr "
+                + ("pos_arg " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+                + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+            )
             self.add_unique_rule(rule, token.kind, uniq_param, customize)
-            self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
+            self.add_unique_rule(
+                "expr ::= async_call", token.kind, uniq_param, customize
+            )
 
-        if opname.startswith('CALL_FUNCTION_VAR'):
+        if opname.startswith("CALL_FUNCTION_VAR"):
             # Python 3.5 changes the stack position of *args. KW args come
             # after *args.
 
@@ -250,43 +262,55 @@ class Python35Parser(Python34Parser):
             # CALL_FUNCTION_VAR_KW with CALL_FUNCTION_EX
 
             token.kind = self.call_fn_name(token)
-            if opname.endswith('KW'):
-                kw = 'expr '
+            if opname.endswith("KW"):
+                kw = "expr "
             else:
-                kw = ''
-            rule = ('call ::= expr expr ' +
-                    ('pos_arg ' * args_pos) +
-                    ('kwarg ' * args_kw) + kw + token.kind)
+                kw = ""
+            rule = (
+                "call ::= expr expr "
+                + ("pos_arg " * args_pos)
+                + ("kwarg " * args_kw)
+                + kw
+                + token.kind
+            )
 
-            # Note: semantic actions make use of the fact of wheter  "args_pos"
+            # Note: semantic actions make use of the fact of whether  "args_pos"
             # zero or not in creating a template rule.
             self.add_unique_rule(rule, token.kind, args_pos, customize)
         else:
-            super(Python35Parser, self).custom_classfunc_rule(opname, token, customize, *args
+            super(Python35Parser, self).custom_classfunc_rule(
+                opname, token, customize, *args
             )
 
 
 class Python35ParserSingle(Python35Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Check grammar
     p = Python35Parser()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
+
     if PYTHON_VERSION_TRIPLE[:2] == (3, 5):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+        opcode_set = set(s.opc.opname).union(
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split()
+            )
+        )
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/parsers/parse36.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2020, 2022 Rocky Bernstein
+#  Copyright (c) 2016-2020, 2022-2024 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,24 +17,25 @@ spark grammar differences over Python 3.5 for Python 3.6.
 """
 from __future__ import print_function
 
-from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse35 import Python35Parser
 from uncompyle6.scanners.tok import Token
 
-class Python36Parser(Python35Parser):
 
+class Python36Parser(Python35Parser):
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
         super(Python36Parser, self).__init__(debug_parser)
         self.customized = {}
 
-
     def p_36_jump(self, args):
         """
         # Zero or one COME_FROM
         # And/or expressions have this
         come_from_opt ::= COME_FROM?
         """
+
     def p_36_misc(self, args):
         """sstmt ::= sstmt RETURN_LAST
 
@@ -58,7 +59,7 @@ class Python36Parser(Python35Parser):
                             come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
 
         # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
-        # RETURN_VALUE is meant. Specifcally this can happen in
+        # RETURN_VALUE is meant. Specifically, this can happen in
         # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
         return             ::= return_expr RETURN_END_IF
         return             ::= return_expr RETURN_VALUE COME_FROM
@@ -190,10 +191,7 @@ class Python36Parser(Python35Parser):
         tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
                                    COME_FROM_FINALLY
 
-        compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
-
-        stmt ::= genexpr_func
-        genexpr_func ::= LOAD_ARG _come_froms FOR_ITER store comp_iter JUMP_BACK
+        compare_chained_right ::= expr COMPARE_OP come_froms JUMP_FORWARD
         """
 
     # Some of this is duplicated from parse37. Eventually we'll probably rebase from
@@ -210,7 +208,8 @@ class Python36Parser(Python35Parser):
         # self.remove_rules("""
         # """)
         super(Python36Parser, self).customize_grammar_rules(tokens, customize)
-        self.remove_rules("""
+        self.remove_rules(
+            """
            _ifstmts_jumpl     ::= c_stmts_opt
            _ifstmts_jumpl     ::= _ifstmts_jump
            except_handler     ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts END_FINALLY COME_FROM
@@ -237,7 +236,8 @@ class Python36Parser(Python35Parser):
                                   for_block pb_ja
                                   else_suite COME_FROM_LOOP
 
-        """)
+        """
+        )
         self.check_reduce["call_kw"] = "AST"
 
         # Opcode names in the custom_ops_processed set have rules that get added
@@ -250,24 +250,23 @@ class Python36Parser(Python35Parser):
         # the start.
         custom_ops_processed = set()
 
-
         for i, token in enumerate(tokens):
             opname = token.kind
 
-            if opname == 'FORMAT_VALUE':
+            if opname == "FORMAT_VALUE":
                 rules_str = """
                     expr              ::= formatted_value1
                     formatted_value1  ::= expr FORMAT_VALUE
                 """
                 self.add_unique_doc_rules(rules_str, customize)
-            elif opname == 'FORMAT_VALUE_ATTR':
+            elif opname == "FORMAT_VALUE_ATTR":
                 rules_str = """
                 expr              ::= formatted_value2
                 formatted_value2  ::= expr expr FORMAT_VALUE_ATTR
                 """
                 self.add_unique_doc_rules(rules_str, customize)
-            elif opname == 'MAKE_FUNCTION_CLOSURE':
-                if 'LOAD_DICTCOMP' in self.seen_ops:
+            elif opname == "MAKE_FUNCTION_CLOSURE":
+                if "LOAD_DICTCOMP" in self.seen_ops:
                     # Is there something general going on here?
                     rule = """
                        dict_comp ::= load_closure LOAD_DICTCOMP LOAD_STR
@@ -275,7 +274,7 @@ class Python36Parser(Python35Parser):
                                      GET_ITER CALL_FUNCTION_1
                        """
                     self.addRule(rule, nop_func)
-                elif 'LOAD_SETCOMP' in self.seen_ops:
+                elif "LOAD_SETCOMP" in self.seen_ops:
                     rule = """
                        set_comp ::= load_closure LOAD_SETCOMP LOAD_STR
                                     MAKE_FUNCTION_CLOSURE expr
@@ -283,7 +282,7 @@ class Python36Parser(Python35Parser):
                        """
                     self.addRule(rule, nop_func)
 
-            elif opname == 'BEFORE_ASYNC_WITH':
+            elif opname == "BEFORE_ASYNC_WITH":
                 rules_str = """
                   stmt ::= async_with_stmt
                   async_with_pre     ::= BEFORE_ASYNC_WITH GET_AWAITABLE LOAD_CONST YIELD_FROM SETUP_ASYNC_WITH
@@ -309,30 +308,37 @@ class Python36Parser(Python35Parser):
                 """
                 self.addRule(rules_str, nop_func)
 
-            elif opname.startswith('BUILD_STRING'):
+            elif opname.startswith("BUILD_STRING"):
                 v = token.attr
                 rules_str = """
                     expr                 ::= joined_str
                     joined_str           ::= %sBUILD_STRING_%d
-                """ % ("expr " * v, v)
+                """ % (
+                    "expr " * v,
+                    v,
+                )
                 self.add_unique_doc_rules(rules_str, customize)
-                if 'FORMAT_VALUE_ATTR' in self.seen_ops:
+                if "FORMAT_VALUE_ATTR" in self.seen_ops:
                     rules_str = """
                       formatted_value_attr ::= expr expr FORMAT_VALUE_ATTR expr BUILD_STRING
                       expr                 ::= formatted_value_attr
                     """
                     self.add_unique_doc_rules(rules_str, customize)
-            elif opname.startswith('BUILD_MAP_UNPACK_WITH_CALL'):
+            elif opname.startswith("BUILD_MAP_UNPACK_WITH_CALL"):
                 v = token.attr
-                rule = 'build_map_unpack_with_call ::= %s%s' % ('expr ' * v, opname)
+                rule = "build_map_unpack_with_call ::= %s%s" % ("expr " * v, opname)
                 self.addRule(rule, nop_func)
-            elif opname.startswith('BUILD_TUPLE_UNPACK_WITH_CALL'):
+            elif opname.startswith("BUILD_TUPLE_UNPACK_WITH_CALL"):
                 v = token.attr
-                rule = ('build_tuple_unpack_with_call ::= ' + 'expr1024 ' * int(v//1024) +
-                        'expr32 ' * int((v//32) % 32) +
-                        'expr ' * (v % 32) + opname)
+                rule = (
+                    "build_tuple_unpack_with_call ::= "
+                    + "expr1024 " * int(v // 1024)
+                    + "expr32 " * int((v // 32) % 32)
+                    + "expr " * (v % 32)
+                    + opname
+                )
                 self.addRule(rule, nop_func)
-                rule = ('starred ::= %s %s' % ('expr ' * v, opname))
+                rule = "starred ::= %s %s" % ("expr " * v, opname)
                 self.addRule(rule, nop_func)
             elif opname == "GET_AITER":
                 self.addRule(
@@ -407,7 +413,7 @@ class Python36Parser(Python35Parser):
                                                 JUMP_LOOP COME_FROM
                                                 POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
 
-                        # FIXME this is a workaround for probalby some bug in the Earley parser
+                        # FIXME this is a workaround for probably some bug in the Earley parser
                         # if we use get_aiter, then list_comp_async doesn't match, and I don't
                         # understand why.
                         expr_get_aiter      ::= expr GET_AITER
@@ -478,7 +484,6 @@ class Python36Parser(Python35Parser):
                 )
                 custom_ops_processed.add(opname)
 
-
             elif opname == "GET_ANEXT":
                 self.addRule(
                     """
@@ -503,7 +508,7 @@ class Python36Parser(Python35Parser):
                 )
                 custom_ops_processed.add(opname)
 
-            elif opname == 'SETUP_ANNOTATIONS':
+            elif opname == "SETUP_ANNOTATIONS":
                 # 3.6 Variable Annotations PEP 526
                 # This seems to come before STORE_ANNOTATION, and doesn't
                 # correspond to direct Python source code.
@@ -519,7 +524,7 @@ class Python36Parser(Python35Parser):
                  """
                 self.addRule(rule, nop_func)
                 # Check to combine assignment + annotation into one statement
-                self.check_reduce['assign'] = 'token'
+                self.check_reduce["assign"] = "token"
             elif opname == "WITH_CLEANUP_START":
                 rules_str = """
                   stmt        ::= with_null
@@ -527,13 +532,13 @@ class Python36Parser(Python35Parser):
                   with_suffix ::= WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
                 """
                 self.addRule(rules_str, nop_func)
-            elif opname == 'SETUP_WITH':
+            elif opname == "SETUP_WITH":
                 rules_str = """
                   with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt COME_FROM_WITH
                                  with_suffix
 
                   # Removes POP_BLOCK LOAD_CONST from 3.6-
-                  withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                  with_as    ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
                                  with_suffix
                   with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
                                  BEGIN_FINALLY COME_FROM_WITH
@@ -545,7 +550,6 @@ class Python36Parser(Python35Parser):
         return
 
     def custom_classfunc_rule(self, opname, token, customize, next_token, is_pypy):
-
         args_pos, args_kw = self.get_pos_kw(token)
 
         # Additional exprs for * and ** args:
@@ -553,140 +557,186 @@ class Python36Parser(Python35Parser):
         #  1 for CALL_FUNCTION_VAR or CALL_FUNCTION_KW
         #  2 for * and ** args (CALL_FUNCTION_VAR_KW).
         # Yes, this computation based on instruction name is a little bit hoaky.
-        nak = ( len(opname)-len('CALL_FUNCTION') ) // 3
+        nak = (len(opname) - len("CALL_FUNCTION")) // 3
         uniq_param = args_kw + args_pos
 
-        if frozenset(('GET_AWAITABLE', 'YIELD_FROM')).issubset(self.seen_ops):
-            rule = ('async_call ::= expr ' +
-                    ('pos_arg ' * args_pos) +
-                    ('kwarg ' * args_kw) +
-                    'expr ' * nak + token.kind +
-                    ' GET_AWAITABLE LOAD_CONST YIELD_FROM')
+        if frozenset(("GET_AWAITABLE", "YIELD_FROM")).issubset(self.seen_ops):
+            rule = (
+                "async_call ::= expr "
+                + ("pos_arg " * args_pos)
+                + ("kwarg " * args_kw)
+                + "expr " * nak
+                + token.kind
+                + " GET_AWAITABLE LOAD_CONST YIELD_FROM"
+            )
             self.add_unique_rule(rule, token.kind, uniq_param, customize)
-            self.add_unique_rule('expr ::= async_call', token.kind, uniq_param, customize)
+            self.add_unique_rule(
+                "expr ::= async_call", token.kind, uniq_param, customize
+            )
 
-        if opname.startswith('CALL_FUNCTION_KW'):
+        if opname.startswith("CALL_FUNCTION_KW"):
             if is_pypy:
                 # PYPY doesn't follow CPython 3.6 CALL_FUNCTION_KW conventions
-                super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
+                super(Python36Parser, self).custom_classfunc_rule(
+                    opname, token, customize, next_token, is_pypy
+                )
             else:
                 self.addRule("expr ::= call_kw36", nop_func)
-                values = 'expr ' * token.attr
-                rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(**locals())
+                values = "expr " * token.attr
+                rule = "call_kw36 ::= expr {values} LOAD_CONST {opname}".format(
+                    **locals()
+                )
                 self.add_unique_rule(rule, token.kind, token.attr, customize)
-        elif opname == 'CALL_FUNCTION_EX_KW':
+        elif opname == "CALL_FUNCTION_EX_KW":
             # Note: this doesn't exist in 3.7 and later
-            self.addRule("""expr        ::= call_ex_kw4
+            self.addRule(
+                """expr        ::= call_ex_kw4
                             call_ex_kw4 ::= expr
                                             expr
                                             expr
                                             CALL_FUNCTION_EX_KW
                          """,
-                         nop_func)
-            if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_op_basenames:
-                self.addRule("""expr        ::= call_ex_kw
+                nop_func,
+            )
+            if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
+                self.addRule(
+                    """expr        ::= call_ex_kw
                                 call_ex_kw  ::= expr expr build_map_unpack_with_call
                                                 CALL_FUNCTION_EX_KW
-                             """, nop_func)
-            if 'BUILD_TUPLE_UNPACK_WITH_CALL' in self.seen_op_basenames:
+                             """,
+                    nop_func,
+                )
+            if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_op_basenames:
                 # FIXME: should this be parameterized by EX value?
-                self.addRule("""expr        ::= call_ex_kw3
+                self.addRule(
+                    """expr        ::= call_ex_kw3
                                 call_ex_kw3 ::= expr
                                                 build_tuple_unpack_with_call
                                                 expr
                                                 CALL_FUNCTION_EX_KW
-                             """, nop_func)
-                if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_op_basenames:
+                             """,
+                    nop_func,
+                )
+                if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_op_basenames:
                     # FIXME: should this be parameterized by EX value?
-                    self.addRule("""expr        ::= call_ex_kw2
+                    self.addRule(
+                        """expr        ::= call_ex_kw2
                                     call_ex_kw2 ::= expr
                                                     build_tuple_unpack_with_call
                                                     build_map_unpack_with_call
                                                     CALL_FUNCTION_EX_KW
-                             """, nop_func)
+                             """,
+                        nop_func,
+                    )
 
-        elif opname == 'CALL_FUNCTION_EX':
-            self.addRule("""
+        elif opname == "CALL_FUNCTION_EX":
+            self.addRule(
+                """
                          expr        ::= call_ex
                          starred     ::= expr
                          call_ex     ::= expr starred CALL_FUNCTION_EX
-                         """, nop_func)
+                         """,
+                nop_func,
+            )
             if self.version >= (3, 6):
-                if 'BUILD_MAP_UNPACK_WITH_CALL' in self.seen_ops:
-                    self.addRule("""
+                if "BUILD_MAP_UNPACK_WITH_CALL" in self.seen_ops:
+                    self.addRule(
+                        """
                             expr        ::= call_ex_kw
                             call_ex_kw  ::= expr expr
                                             build_map_unpack_with_call CALL_FUNCTION_EX
-                            """, nop_func)
-                if 'BUILD_TUPLE_UNPACK_WITH_CALL' in self.seen_ops:
-                    self.addRule("""
+                            """,
+                        nop_func,
+                    )
+                if "BUILD_TUPLE_UNPACK_WITH_CALL" in self.seen_ops:
+                    self.addRule(
+                        """
                             expr        ::= call_ex_kw3
                             call_ex_kw3 ::= expr
                                             build_tuple_unpack_with_call
                                             %s
                                             CALL_FUNCTION_EX
-                            """ % 'expr ' * token.attr, nop_func)
+                            """
+                        % "expr "
+                        * token.attr,
+                        nop_func,
+                    )
                     pass
 
                 # FIXME: Is this right?
-                self.addRule("""
+                self.addRule(
+                    """
                             expr        ::= call_ex_kw4
                             call_ex_kw4 ::= expr
                                             expr
                                             expr
                                             CALL_FUNCTION_EX
-                            """, nop_func)
+                            """,
+                    nop_func,
+                )
             pass
         else:
-            super(Python36Parser, self).custom_classfunc_rule(opname, token, customize, next_token, is_pypy)
+            super(Python36Parser, self).custom_classfunc_rule(
+                opname, token, customize, next_token, is_pypy
+            )
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
-        invalid = super(Python36Parser,
-                        self).reduce_is_invalid(rule, ast,
-                                                tokens, first, last)
+        invalid = super(Python36Parser, self).reduce_is_invalid(
+            rule, ast, tokens, first, last
+        )
         if invalid:
             return invalid
-        if rule[0] == 'assign':
+        if rule[0] == "assign":
             # Try to combine assignment + annotation into one statement
-            if (len(tokens) >= last + 1 and
-                tokens[last] == 'LOAD_NAME' and
-                tokens[last+1] == 'STORE_ANNOTATION' and
-                tokens[last-1].pattr == tokens[last+1].pattr):
+            if (
+                len(tokens) >= last + 1
+                and tokens[last] == "LOAD_NAME"
+                and tokens[last + 1] == "STORE_ANNOTATION"
+                and tokens[last - 1].pattr == tokens[last + 1].pattr
+            ):
                 # Will handle as ann_assign_init_value
                 return True
             pass
-        if rule[0] == 'call_kw':
+        if rule[0] == "call_kw":
             # Make sure we don't derive call_kw
             nt = ast[0]
             while not isinstance(nt, Token):
-                if nt[0] == 'call_kw':
+                if nt[0] == "call_kw":
                     return True
                 nt = nt[0]
                 pass
             pass
         return False
 
+
 class Python36ParserSingle(Python36Parser, PythonParserSingle):
     pass
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     # Check grammar
     p = Python36Parser()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
+
     if PYTHON_VERSION_TRIPLE[:2] == (3, 6):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
         from uncompyle6.scanner import get_scanner
+
         s = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY)
-        opcode_set = set(s.opc.opname).union(set(
-            """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
+        opcode_set = set(s.opc.opname).union(
+            set(
+                """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split()
+            )
+        )
         remain_tokens = set(tokens) - opcode_set
         import re
-        remain_tokens = set([re.sub(r'_\d+$', '', t) for t in remain_tokens])
-        remain_tokens = set([re.sub('_CONT$', '', t) for t in remain_tokens])
+
+        remain_tokens = set([re.sub(r"_\d+$", "", t) for t in remain_tokens])
+        remain_tokens = set([re.sub("_CONT$", "", t) for t in remain_tokens])
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         # print(sorted(p.rule2name.items()))

uncompyle6/parsers/parse37.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2020, 2022 Rocky Bernstein
+#  Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,10 +17,12 @@ Python 3.7 grammar for the spark Earley-algorithm parser.
 """
 from __future__ import print_function
 
-from uncompyle6.scanners.tok import Token
-from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse37base import Python37BaseParser
+from uncompyle6.scanners.tok import Token
+
 
 class Python37Parser(Python37BaseParser):
     def __init__(self, debug_parser=PARSER_DEFAULT_DEBUG):
@@ -62,6 +64,9 @@ class Python37Parser(Python37BaseParser):
         c_stmts ::= lastc_stmt
         c_stmts ::= continues
 
+        ending_return  ::= RETURN_VALUE RETURN_LAST
+        ending_return  ::= RETURN_VALUE_LAMBDA LAMBDA_MARKER
+
         lastc_stmt ::= iflaststmt
         lastc_stmt ::= forelselaststmt
         lastc_stmt ::= ifelsestmtc
@@ -130,7 +135,8 @@ class Python37Parser(Python37BaseParser):
         stmt   ::= return
         return ::= return_expr RETURN_VALUE
 
-        # "returns" nonterminal is a sequence of statements that ends in a RETURN statement.
+        # "returns" nonterminal is a sequence of statements that ends in a
+        # RETURN statement.
         # In later Python versions with jump optimization, this can cause JUMPs
         # that would normally appear to be omitted.
 
@@ -220,11 +226,11 @@ class Python37Parser(Python37BaseParser):
         compare        ::= compare_single
         compare_single ::= expr expr COMPARE_OP
 
-        # A compare_chained is two comparisions like x <= y <= z
-        compare_chained  ::= expr compare_chained1 ROT_TWO POP_TOP _come_froms
-        compare_chained2 ::= expr COMPARE_OP JUMP_FORWARD
+        # A compare_chained is two comparisons like x <= y <= z
+        compare_chained  ::= expr compared_chained_middle ROT_TWO POP_TOP _come_froms
+        compare_chained_right ::= expr COMPARE_OP JUMP_FORWARD
 
-        # Non-null kvlist items are broken out in the indiviual grammars
+        # Non-null kvlist items are broken out in the individual grammars
         kvlist ::=
 
         # Positional arguments in make_function
@@ -245,8 +251,7 @@ class Python37Parser(Python37BaseParser):
         """
 
     def p_generator_exp(self, args):
-        """
-        """
+        """ """
 
     def p_jump(self, args):
         """
@@ -439,10 +444,10 @@ class Python37Parser(Python37BaseParser):
         """
         if_exp::= expr jmp_false expr jump_forward_else expr COME_FROM
 
-        # compare_chained2 is used in a "chained_compare": x <= y <= z
+        # compare_chained_right is used in a "chained_compare": x <= y <= z
         # used exclusively in compare_chained
-        compare_chained2 ::= expr COMPARE_OP RETURN_VALUE
-        compare_chained2 ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
+        compare_chained_right ::= expr COMPARE_OP RETURN_VALUE
+        compare_chained_right ::= expr COMPARE_OP RETURN_VALUE_LAMBDA
 
         # Python < 3.5 no POP BLOCK
         whileTruestmt  ::= SETUP_LOOP l_stmts_opt JUMP_BACK COME_FROM_LOOP
@@ -554,7 +559,7 @@ class Python37Parser(Python37BaseParser):
         ifelsestmtl ::= testexpr_cf c_stmts_opt jb_else else_suitel
 
         # 3.5 Has jump optimization which can route the end of an
-        # "if/then" back to to a loop just before an else.
+        # "if/then" back to a loop just before an else.
         jump_absolute_else ::= jb_else
         jump_absolute_else ::= CONTINUE ELSE
 
@@ -625,41 +630,41 @@ class Python37Parser(Python37BaseParser):
         compare_chained     ::= compare_chained37
         compare_chained     ::= compare_chained37_false
 
-        compare_chained37   ::= expr compare_chained1a_37
-        compare_chained37   ::= expr compare_chained1c_37
+        compare_chained37   ::= expr compared_chained_middlea_37
+        compare_chained37   ::= expr compared_chained_middlec_37
 
-        compare_chained37_false  ::= expr compare_chained1_false_37
-        compare_chained37_false  ::= expr compare_chained1b_false_37
-        compare_chained37_false  ::= expr compare_chained2_false_37
+        compare_chained37_false  ::= expr compared_chained_middle_false_37
+        compare_chained37_false  ::= expr compared_chained_middleb_false_37
+        compare_chained37_false  ::= expr compare_chained_right_false_37
 
-        compare_chained1a_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-        compare_chained1a_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                      compare_chained2a_37 COME_FROM POP_TOP COME_FROM
-        compare_chained1b_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                      compare_chained2b_false_37 POP_TOP _jump COME_FROM
+        compared_chained_middlea_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+        compared_chained_middlea_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained_righta_37 COME_FROM POP_TOP COME_FROM
+        compared_chained_middleb_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained_rightb_false_37 POP_TOP _jump COME_FROM
 
-        compare_chained1c_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                      compare_chained2a_37 POP_TOP
+        compared_chained_middlec_37      ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained_righta_37 POP_TOP
 
-        compare_chained1_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                      compare_chained2c_37 POP_TOP JUMP_FORWARD COME_FROM
-        compare_chained1_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                      compare_chained2b_false_37 POP_TOP _jump COME_FROM
+        compared_chained_middle_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained_rightc_37 POP_TOP JUMP_FORWARD COME_FROM
+        compared_chained_middle_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained_rightb_false_37 POP_TOP _jump COME_FROM
 
-        compare_chained2_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
-                                      compare_chained2a_false_37 POP_TOP JUMP_BACK COME_FROM
+        compare_chained_right_false_37 ::= expr DUP_TOP ROT_THREE COMPARE_OP POP_JUMP_IF_FALSE
+                                      compare_chained_righta_false_37 POP_TOP JUMP_BACK COME_FROM
 
-        compare_chained2a_37       ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_FORWARD
-        compare_chained2a_37       ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_BACK
-        compare_chained2a_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE jf_cfs
+        compare_chained_righta_37       ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_FORWARD
+        compare_chained_righta_37       ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_TRUE JUMP_BACK
+        compare_chained_righta_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE jf_cfs
 
-        compare_chained2b_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD COME_FROM
-        compare_chained2b_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD
+        compare_chained_rightb_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD COME_FROM
+        compare_chained_rightb_false_37 ::= expr COMPARE_OP come_from_opt POP_JUMP_IF_FALSE JUMP_FORWARD
 
-        compare_chained2c_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
-                                       compare_chained2a_false_37 ELSE
-        compare_chained2c_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
-                                       compare_chained2a_false_37
+        compare_chained_rightc_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
+                                       compare_chained_righta_false_37 ELSE
+        compare_chained_rightc_37       ::= expr DUP_TOP ROT_THREE COMPARE_OP come_from_opt POP_JUMP_IF_FALSE
+                                       compare_chained_righta_false_37
         """
 
     def p_37_conditionals(self, args):
@@ -667,11 +672,13 @@ class Python37Parser(Python37BaseParser):
         expr                       ::= if_exp37
         if_exp37                   ::= expr expr jf_cfs expr COME_FROM
         jf_cfs                     ::= JUMP_FORWARD _come_froms
-        ifelsestmt                 ::= testexpr c_stmts_opt jf_cfs else_suite opt_come_from_except
+        ifelsestmt                 ::= testexpr c_stmts_opt jf_cfs else_suite
+                                       opt_come_from_except
 
         # This is probably more realistically an "ifstmt" (with a null else)
         # see _cmp() of python3.8/distutils/__pycache__/version.cpython-38.opt-1.pyc
-        ifelsestmt                 ::= testexpr stmts jf_cfs else_suite_opt opt_come_from_except
+        ifelsestmt                 ::= testexpr stmts jf_cfs else_suite_opt
+                                       opt_come_from_except
 
 
         expr_pjit                  ::= expr POP_JUMP_IF_TRUE
@@ -694,7 +701,8 @@ class Python37Parser(Python37BaseParser):
         expr                       ::= if_exp_37a
         expr                       ::= if_exp_37b
         if_exp_37a                 ::= and_not expr JUMP_FORWARD come_froms expr COME_FROM
-        if_exp_37b                 ::= expr jmp_false expr POP_JUMP_IF_FALSE jump_forward_else expr
+        if_exp_37b                 ::= expr jmp_false expr POP_JUMP_IF_FALSE
+                                       jump_forward_else expr
         jmp_false_cf               ::= POP_JUMP_IF_FALSE COME_FROM
         comp_if                    ::= or jmp_false_cf comp_iter
         """
@@ -735,11 +743,11 @@ class Python37Parser(Python37BaseParser):
 
         stmt ::= set_comp_func
 
+        # TODO: simplify this
         set_comp_func ::= BUILD_SET_0 LOAD_ARG for_iter store comp_iter
-                          JUMP_BACK RETURN_VALUE RETURN_LAST
-
+                          JUMP_BACK ending_return
         set_comp_func ::= BUILD_SET_0 LOAD_ARG for_iter store comp_iter
-                          COME_FROM JUMP_BACK RETURN_VALUE RETURN_LAST
+                          COME_FROM JUMP_BACK ending_return
 
         comp_body ::= dict_comp_body
         comp_body ::= set_comp_body
@@ -750,11 +758,12 @@ class Python37Parser(Python37BaseParser):
         """
 
     def p_dict_comp3(self, args):
-        """"
+        """ "
         expr ::= dict_comp
         stmt ::= dict_comp_func
+
         dict_comp_func ::= BUILD_MAP_0 LOAD_ARG for_iter store
-                           comp_iter JUMP_BACK RETURN_VALUE RETURN_LAST
+                           comp_iter JUMP_BACK ending_return
 
         comp_iter     ::= comp_if
         comp_iter     ::= comp_if_not
@@ -1014,11 +1023,11 @@ class Python37Parser(Python37BaseParser):
         and ::= expr jmp_false expr COME_FROM
         or  ::= expr_jt  expr COME_FROM
 
-        # compare_chained1 is used exclusively in chained_compare
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained1 COME_FROM
-        compare_chained1 ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
-                             compare_chained2 COME_FROM
+        # compared_chained_middle is used exclusively in chained_compare
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                                    compared_chained_middle COME_FROM
+        compared_chained_middle ::= expr DUP_TOP ROT_THREE COMPARE_OP JUMP_IF_FALSE_OR_POP
+                                    compare_chained_right COME_FROM
         """
 
     def p_stmt3(self, args):
@@ -1136,7 +1145,7 @@ class Python37Parser(Python37BaseParser):
                             come_froms JUMP_BACK come_froms POP_BLOCK COME_FROM_LOOP
 
         # 3.6 due to jump optimization, we sometimes add RETURN_END_IF where
-        # RETURN_VALUE is meant. Specifcally this can happen in
+        # RETURN_VALUE is meant. Specifically this can happen in
         # ifelsestmt -> ...else_suite _. suite_stmts... (last) stmt
         return ::= return_expr RETURN_END_IF
         return ::= return_expr RETURN_VALUE COME_FROM
@@ -1205,7 +1214,7 @@ class Python37Parser(Python37BaseParser):
         tryfinally_return_stmt ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK LOAD_CONST
                                    COME_FROM_FINALLY
 
-        compare_chained2 ::= expr COMPARE_OP come_froms JUMP_FORWARD
+        compare_chained_right ::= expr COMPARE_OP come_froms JUMP_FORWARD
         """
 
     def p_37_misc(self, args):
@@ -1369,7 +1378,7 @@ class Python37Parser(Python37BaseParser):
                                                 JUMP_BACK COME_FROM
                                                 POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_TOP
 
-                        # FIXME this is a workaround for probalby some bug in the Earley parser
+                        # FIXME this is a workaround for probably some bug in the Earley parser
                         # if we use get_aiter, then list_comp_async doesn't match, and I don't
                         # understand why.
                         expr_get_aiter      ::= expr GET_AITER
@@ -1546,7 +1555,7 @@ class Python37Parser(Python37BaseParser):
                                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
 
                 # Removes POP_BLOCK LOAD_CONST from 3.6-
-                withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                with_as    ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
                                WITH_CLEANUP_START WITH_CLEANUP_FINISH END_FINALLY
                 """
                 if self.version < (3, 8):
@@ -1567,7 +1576,6 @@ class Python37Parser(Python37BaseParser):
             pass
 
     def custom_classfunc_rule(self, opname, token, customize, next_token):
-
         args_pos, args_kw = self.get_pos_kw(token)
 
         # Additional exprs for * and ** args:
@@ -1710,6 +1718,7 @@ class Python37Parser(Python37BaseParser):
             pass
         return False
 
+
 def info(args):
     # Check grammar
     p = Python37Parser()
@@ -1740,7 +1749,7 @@ if __name__ == "__main__":
     # FIXME: DRY this with other parseXX.py routines
     p = Python37Parser()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
 
     if PYTHON_VERSION_TRIPLE[:2] == (3, 7):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()

uncompyle6/parsers/parse37base.py

@@ -1,12 +1,11 @@
-#  Copyright (c) 2016-2017, 2019-2020, 2022-2023 Rocky Bernstein
+#  Copyright (c) 2016-2017, 2019-2020, 2022-2024 Rocky Bernstein
 """
 Python 3.7 base code. We keep non-custom-generated grammar rules out of this file.
 """
-from uncompyle6.parser import ParserError, PythonParser, nop_func
-from uncompyle6.parsers.treenode import SyntaxTree
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
 from spark_parser.spark import rule2str
 
+from uncompyle6.parser import ParserError, PythonParser, nop_func
 from uncompyle6.parsers.reducecheck import (
     and_invalid,
     ifelsestmt,
@@ -16,9 +15,10 @@ from uncompyle6.parsers.reducecheck import (
     or_check,
     testtrue,
     tryelsestmtl3,
-    while1stmt,
     while1elsestmt,
+    while1stmt,
 )
+from uncompyle6.parsers.treenode import SyntaxTree
 
 
 class Python37BaseParser(PythonParser):
@@ -38,7 +38,7 @@ class Python37BaseParser(PythonParser):
             return "%s_0" % (token.kind)
 
     def add_make_function_rule(self, rule, opname, attr, customize):
-        """Python 3.3 added a an addtional LOAD_STR before MAKE_FUNCTION and
+        """Python 3.3 added a an additional LOAD_STR before MAKE_FUNCTION and
         this has an effect on many rules.
         """
         new_rule = rule % "LOAD_STR "
@@ -54,7 +54,7 @@ class Python37BaseParser(PythonParser):
                         expr
                         call
                         CALL_FUNCTION_3
-         """
+        """
         # FIXME: I bet this can be simplified
         # look for next MAKE_FUNCTION
         for i in range(i + 1, len(tokens)):
@@ -104,7 +104,6 @@ class Python37BaseParser(PythonParser):
     # organization for this. For example, arrange organize by opcode base?
 
     def customize_grammar_rules(self, tokens, customize):
-
         is_pypy = False
 
         # For a rough break out on the first word. This may
@@ -321,18 +320,24 @@ class Python37BaseParser(PythonParser):
 
             elif opname in ("BUILD_CONST_LIST", "BUILD_CONST_DICT", "BUILD_CONST_SET"):
                 if opname == "BUILD_CONST_DICT":
-                    rule = f"""
+                    rule = (
+                        """
                             add_consts          ::= ADD_VALUE*
-                            const_list          ::= COLLECTION_START add_consts {opname}
+                            const_list          ::= COLLECTION_START add_consts %s
                             dict                ::= const_list
                             expr                ::= dict
                         """
+                        % opname
+                    )
                 else:
-                    rule = f"""
+                    rule = (
+                        """
                             add_consts          ::= ADD_VALUE*
-                            const_list          ::= COLLECTION_START add_consts {opname}
+                            const_list          ::= COLLECTION_START add_consts %s
                             expr                ::= const_list
                         """
+                        % opname
+                    )
                 self.addRule(rule, nop_func)
 
             elif opname_base == "BUILD_CONST_KEY_MAP":
@@ -348,7 +353,6 @@ class Python37BaseParser(PythonParser):
                 self.addRule(rule, nop_func)
 
             elif opname_base in ("BUILD_MAP", "BUILD_MAP_UNPACK"):
-
                 if opname == "BUILD_MAP_UNPACK":
                     self.addRule(
                         """
@@ -525,7 +529,6 @@ class Python37BaseParser(PythonParser):
                     "CALL_FUNCTION_VAR_KW",
                 )
             ) or opname.startswith("CALL_FUNCTION_KW"):
-
                 if opname == "CALL_FUNCTION" and token.attr == 1:
                     rule = """
                      expr         ::= dict_comp
@@ -720,7 +723,9 @@ class Python37BaseParser(PythonParser):
                 )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_LISTCOMP":
-                self.add_unique_rule("expr ::= listcomp", opname, token.attr, customize)
+                self.add_unique_rule(
+                    "expr ::= list_comp", opname, token.attr, customize
+                )
                 custom_ops_processed.add(opname)
             elif opname == "LOAD_NAME":
                 if (
@@ -799,7 +804,7 @@ class Python37BaseParser(PythonParser):
                             #   and have GET_ITER CALL_FUNCTION_1
                             # Todo: For Pypy we need to modify this slightly
                             rule_pat = (
-                                "listcomp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
+                                "list_comp ::= %sload_closure LOAD_LISTCOMP %%s%s expr "
                                 "GET_ITER CALL_FUNCTION_1"
                                 % ("pos_arg " * args_pos, opname)
                             )
@@ -897,14 +902,14 @@ class Python37BaseParser(PythonParser):
                         # 'exprs' in the rule above into a
                         # tuple.
                         rule_pat = (
-                            "listcomp ::= load_closure LOAD_LISTCOMP %%s%s "
+                            "list_comp ::= load_closure LOAD_LISTCOMP %%s%s "
                             "expr GET_ITER CALL_FUNCTION_1" % (opname,)
                         )
                         self.add_make_function_rule(
                             rule_pat, opname, token.attr, customize
                         )
                         rule_pat = (
-                            "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
                         )
                         self.add_make_function_rule(
@@ -938,7 +943,7 @@ class Python37BaseParser(PythonParser):
                         #   and have GET_ITER CALL_FUNCTION_1
                         # Todo: For Pypy we need to modify this slightly
                         rule_pat = (
-                            "listcomp ::= %sLOAD_LISTCOMP %%s%s expr "
+                            "list_comp ::= %sLOAD_LISTCOMP %%s%s expr "
                             "GET_ITER CALL_FUNCTION_1" % ("expr " * args_pos, opname)
                         )
                         self.add_make_function_rule(
@@ -1056,14 +1061,14 @@ class Python37BaseParser(PythonParser):
             elif opname == "SETUP_WITH":
                 rules_str = """
                   stmt       ::= with
-                  stmt       ::= withasstmt
+                  stmt       ::= with_as
 
                   with       ::= expr
                                  SETUP_WITH POP_TOP
                                  suite_stmts_opt
                                  COME_FROM_WITH
                                  with_suffix
-                  withasstmt ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
+                  with_as ::= expr SETUP_WITH store suite_stmts_opt COME_FROM_WITH
                                  with_suffix
 
                   with       ::= expr
@@ -1072,7 +1077,7 @@ class Python37BaseParser(PythonParser):
                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
                                  with_suffix
 
-                  withasstmt ::= expr
+                  with_as ::= expr
                                  SETUP_WITH store suite_stmts_opt
                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
                                  with_suffix
@@ -1081,7 +1086,7 @@ class Python37BaseParser(PythonParser):
                                  SETUP_WITH POP_TOP suite_stmts_opt
                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
                                  with_suffix
-                  withasstmt ::= expr
+                  with_as ::= expr
                                  SETUP_WITH store suite_stmts_opt
                                  POP_BLOCK LOAD_CONST COME_FROM_WITH
                                  with_suffix
@@ -1099,17 +1104,18 @@ class Python37BaseParser(PythonParser):
                                      POP_BLOCK LOAD_CONST COME_FROM_WITH
                                      with_suffix
 
-                      withasstmt ::= expr
-                                     SETUP_WITH store suite_stmts_opt
-                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
-
-                      withasstmt ::= expr
-                                     SETUP_WITH store suite_stmts
-                                     POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
 
                       with       ::= expr SETUP_WITH POP_TOP suite_stmts_opt POP_BLOCK
                                      BEGIN_FINALLY COME_FROM_WITH
                                      with_suffix
+
+                      with_as    ::= expr
+                                     SETUP_WITH store suite_stmts_opt
+                                     POP_BLOCK LOAD_CONST COME_FROM_WITH
+
+                      with_as    ::= expr
+                                    SETUP_WITH store suite_stmts
+                                    POP_BLOCK BEGIN_FINALLY COME_FROM_WITH with_suffix
                     """
                 self.addRule(rules_str, nop_func)
 
@@ -1259,12 +1265,18 @@ class Python37BaseParser(PythonParser):
             if fn:
                 return fn(self, lhs, n, rule, ast, tokens, first, last)
         except Exception:
-            import sys, traceback
+            import sys
+            import traceback
 
             print(
-                f"Exception in {fn.__name__} {sys.exc_info()[1]}\n"
-                + f"rule: {rule2str(rule)}\n"
-                + f"offsets {tokens[first].offset} .. {tokens[last].offset}"
+                ("Exception in %s %s\n" + "rule: %s\n" + "offsets %s .. %s")
+                % (
+                    fn.__name__,
+                    sys.exc_info()[1],
+                    rule2str(rule),
+                    tokens[first].offset,
+                    tokens[last].offset,
+                )
             )
             print(traceback.print_tb(sys.exc_info()[2], -1))
             raise ParserError(tokens[last], tokens[last].off2int(), self.debug["rules"])

uncompyle6/parsers/parse38.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2017-2020, 2022 Rocky Bernstein
+#  Copyright (c) 2017-2020, 2022-2024 Rocky Bernstein
 #
 #  This program is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -17,267 +17,272 @@ spark grammar differences over Python 3.7 for Python 3.8
 """
 from __future__ import print_function
 
-from uncompyle6.parser import PythonParserSingle, nop_func
 from spark_parser import DEFAULT_DEBUG as PARSER_DEFAULT_DEBUG
+
+from uncompyle6.parser import PythonParserSingle, nop_func
 from uncompyle6.parsers.parse37 import Python37Parser
+from uncompyle6.parsers.reducecheck.pop_return import pop_return_check
+
 
 class Python38Parser(Python37Parser):
     def p_38_stmt(self, args):
         """
-        stmt               ::= async_for_stmt38
-        stmt               ::= async_forelse_stmt38
-        stmt               ::= call_stmt
-        stmt               ::= continue
-        stmt               ::= for38
-        stmt               ::= forelselaststmt38
-        stmt               ::= forelselaststmtl38
-        stmt               ::= forelsestmt38
-        stmt               ::= try_elsestmtl38
-        stmt               ::= try_except38
-        stmt               ::= try_except38r
-        stmt               ::= try_except38r2
-        stmt               ::= try_except38r3
-        stmt               ::= try_except38r4
-        stmt               ::= try_except_as
-        stmt               ::= try_except_ret38
-        stmt               ::= tryfinally38astmt
-        stmt               ::= tryfinally38rstmt
-        stmt               ::= tryfinally38rstmt2
-        stmt               ::= tryfinally38rstmt3
-        stmt               ::= tryfinally38stmt
-        stmt               ::= whileTruestmt38
-        stmt               ::= whilestmt38
+         stmt               ::= async_for_stmt38
+         stmt               ::= async_forelse_stmt38
+         stmt               ::= call_stmt
+         stmt               ::= continue
+         stmt               ::= for38
+         stmt               ::= forelselaststmt38
+         stmt               ::= forelselaststmtl38
+         stmt               ::= forelsestmt38
+         stmt               ::= try_elsestmtl38
+         stmt               ::= try_except38
+         stmt               ::= try_except38r
+         stmt               ::= try_except38r2
+         stmt               ::= try_except38r3
+         stmt               ::= try_except38r4
+         stmt               ::= try_except_as
+         stmt               ::= try_except_ret38
+         stmt               ::= tryfinally38astmt
+         stmt               ::= tryfinally38rstmt
+         stmt               ::= tryfinally38rstmt2
+         stmt               ::= tryfinally38rstmt3
+         stmt               ::= tryfinally38stmt
+         stmt               ::= whileTruestmt38
+         stmt               ::= whilestmt38
 
-        call_stmt          ::= call
-        break ::= POP_BLOCK BREAK_LOOP
-        break ::= POP_BLOCK POP_TOP BREAK_LOOP
-        break ::= POP_TOP BREAK_LOOP
-        break ::= POP_EXCEPT BREAK_LOOP
+         call_stmt          ::= call
+         break ::= POP_BLOCK BREAK_LOOP
+         break ::= POP_BLOCK POP_TOP BREAK_LOOP
+         break ::= POP_TOP BREAK_LOOP
+         break ::= POP_EXCEPT BREAK_LOOP
 
-        # The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed.
-        # Inside an loop we can have this, which can only appear in side a try/except
-        # And it can also appear at the end of the try except.
-        continue           ::= POP_EXCEPT JUMP_BACK
+         # The "continue" rule is a weird one. In 3.8, CONTINUE_LOOP was removed.
+         # Inside an loop we can have this, which can only appear in side a try/except
+         # And it can also appear at the end of the try except.
+         continue           ::= POP_EXCEPT JUMP_BACK
 
 
-        # FIXME: this should be restricted to being inside a try block
-        stmt               ::= except_ret38
-        stmt               ::= except_ret38a
+         # FIXME: this should be restricted to being inside a try block
+         stmt               ::= except_ret38
+         stmt               ::= except_ret38a
 
-        # FIXME: this should be added only when seeing GET_AITER or YIELD_FROM
-        async_for          ::= GET_AITER _come_froms
-                               SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
-        async_for_stmt38   ::= expr async_for
-                               store for_block
-                               COME_FROM_FINALLY
-                               END_ASYNC_FOR
+         # FIXME: this should be added only when seeing GET_AITER or YIELD_FROM
+         async_for          ::= GET_AITER _come_froms
+                                SETUP_FINALLY GET_ANEXT LOAD_CONST YIELD_FROM POP_BLOCK
+         async_for_stmt38   ::= expr async_for
+                                store for_block
+                                COME_FROM_FINALLY
+                                END_ASYNC_FOR
 
-       genexpr_func_async  ::= LOAD_ARG func_async_prefix
-                               store comp_iter
-                               JUMP_BACK COME_FROM_FINALLY
-                               END_ASYNC_FOR
+        genexpr_func_async  ::= LOAD_ARG func_async_prefix
+                                store comp_iter
+                                JUMP_BACK COME_FROM_FINALLY
+                                END_ASYNC_FOR
 
-        # FIXME: come froms after the else_suite or END_ASYNC_FOR distinguish which of
-        # for / forelse is used. Add come froms and check of add up control-flow detection phase.
-        async_forelse_stmt38 ::= expr
-                               GET_AITER
-                               SETUP_FINALLY
-                               GET_ANEXT
-                               LOAD_CONST
-                               YIELD_FROM
-                               POP_BLOCK
-                               store for_block
-                               COME_FROM_FINALLY
-                               END_ASYNC_FOR
-                               else_suite
+         # FIXME: "come_froms" after the "else_suite" or END_ASYNC_FOR distinguish which of
+         # for / forelse is used. Add "come_froms" and check of add up control-flow detection phase.
+         async_forelse_stmt38 ::= expr
+                                GET_AITER
+                                SETUP_FINALLY
+                                GET_ANEXT
+                                LOAD_CONST
+                                YIELD_FROM
+                                POP_BLOCK
+                                store for_block
+                                COME_FROM_FINALLY
+                                END_ASYNC_FOR
+                                else_suite
 
-        # Seems to be used to discard values before a return in a "for" loop
-        discard_top        ::= ROT_TWO POP_TOP
-        discard_tops       ::= discard_top+
+         # Seems to be used to discard values before a return in a "for" loop
+         discard_top        ::= ROT_TWO POP_TOP
+         discard_tops       ::= discard_top+
 
-        return             ::= return_expr
-                               discard_tops RETURN_VALUE
+         return             ::= return_expr
+                                discard_tops RETURN_VALUE
 
-        return             ::= popb_return
-        return             ::= pop_return
-        return             ::= pop_ex_return
-        except_stmt        ::= pop_ex_return
-        pop_return         ::= POP_TOP return_expr RETURN_VALUE
-        popb_return        ::= return_expr POP_BLOCK RETURN_VALUE
-        pop_ex_return      ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
+         return             ::= popb_return
+         return             ::= pop_return
+         return             ::= pop_ex_return
+         except_stmt        ::= pop_ex_return
+         pop_return         ::= POP_TOP return_expr RETURN_VALUE
+         popb_return        ::= return_expr POP_BLOCK RETURN_VALUE
+         pop_ex_return      ::= return_expr ROT_FOUR POP_EXCEPT RETURN_VALUE
 
-        # 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
-        lastl_stmt         ::= ifpoplaststmtl
-        ifpoplaststmtl     ::= testexpr POP_TOP c_stmts_opt
-        ifelsestmtl        ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms
+         # 3.8 can push a looping JUMP_BACK into into a JUMP_ from a statement that jumps to it
+         lastl_stmt         ::= ifpoplaststmtl
+         ifpoplaststmtl     ::= testexpr POP_TOP c_stmts_opt
+         ifelsestmtl        ::= testexpr c_stmts_opt jb_cfs else_suitel JUMP_BACK come_froms
 
-        # Keep indices the same in ifelsestmtl
-        cf_pt              ::= COME_FROM POP_TOP
-        ifelsestmtl        ::= testexpr c_stmts cf_pt else_suite
+         # Keep indices the same in ifelsestmtl
+         cf_pt              ::= COME_FROM POP_TOP
+         ifelsestmtl        ::= testexpr c_stmts cf_pt else_suite
 
-        for38              ::= expr get_iter store for_block JUMP_BACK
-        for38              ::= expr get_for_iter store for_block JUMP_BACK
-        for38              ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK
-        for38              ::= expr get_for_iter store for_block
+         for38              ::= expr get_iter store for_block JUMP_BACK
+         for38              ::= expr get_for_iter store for_block JUMP_BACK
+         for38              ::= expr get_for_iter store for_block JUMP_BACK POP_BLOCK
+         for38              ::= expr get_for_iter store for_block
 
-        forelsestmt38      ::= expr get_for_iter store for_block POP_BLOCK else_suite
-        forelsestmt38      ::= expr get_for_iter store for_block JUMP_BACK _come_froms else_suite
+         forelsestmt38      ::= expr get_for_iter store for_block POP_BLOCK else_suite
+         forelsestmt38      ::= expr get_for_iter store for_block JUMP_BACK _come_froms
+                                else_suite
 
-        forelselaststmt38  ::= expr get_for_iter store for_block POP_BLOCK else_suitec
-        forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
+         forelselaststmt38  ::= expr get_for_iter store for_block POP_BLOCK else_suitec
+         forelselaststmtl38 ::= expr get_for_iter store for_block POP_BLOCK else_suitel
 
-        returns_in_except   ::= _stmts except_return_value
-        except_return_value ::= POP_BLOCK return
-        except_return_value ::= expr POP_BLOCK RETURN_VALUE
+         returns_in_except   ::= _stmts except_return_value
+         except_return_value ::= POP_BLOCK return
+         except_return_value ::= expr POP_BLOCK RETURN_VALUE
 
-        whilestmt38        ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK POP_BLOCK
-        whilestmt38        ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK
-        whilestmt38        ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms
-        whilestmt38        ::= _come_froms testexpr returns               POP_BLOCK
-        whilestmt38        ::= _come_froms testexpr l_stmts     JUMP_BACK
-        whilestmt38        ::= _come_froms testexpr l_stmts     come_froms
+         whilestmt38        ::= _come_froms testexpr l_stmts_opt COME_FROM JUMP_BACK
+                                 POP_BLOCK
+         whilestmt38        ::= _come_froms testexpr l_stmts_opt JUMP_BACK POP_BLOCK
+         whilestmt38        ::= _come_froms testexpr l_stmts_opt JUMP_BACK come_froms
+         whilestmt38        ::= _come_froms testexpr returns               POP_BLOCK
+         whilestmt38        ::= _come_froms testexpr l_stmts     JUMP_BACK
+         whilestmt38        ::= _come_froms testexpr l_stmts     come_froms
 
-        # while1elsestmt   ::=          l_stmts     JUMP_BACK
-        whileTruestmt      ::= _come_froms l_stmts              JUMP_BACK POP_BLOCK
-        while1stmt         ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
-        whileTruestmt38    ::= _come_froms l_stmts JUMP_BACK
-        whileTruestmt38    ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE
-        whileTruestmt38    ::= _come_froms pass JUMP_BACK
+         # while1elsestmt   ::=          l_stmts     JUMP_BACK
+         whileTruestmt      ::= _come_froms l_stmts              JUMP_BACK POP_BLOCK
+         while1stmt         ::= _come_froms l_stmts COME_FROM JUMP_BACK COME_FROM_LOOP
+         whileTruestmt38    ::= _come_froms l_stmts JUMP_BACK
+         whileTruestmt38    ::= _come_froms l_stmts JUMP_BACK COME_FROM_EXCEPT_CLAUSE
+         whileTruestmt38    ::= _come_froms pass JUMP_BACK
 
-        for_block          ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK
+         for_block          ::= _come_froms l_stmts_opt _come_from_loops JUMP_BACK
 
-        except_cond1       ::= DUP_TOP expr COMPARE_OP jmp_false
-                               POP_TOP POP_TOP POP_TOP
-                               POP_EXCEPT
-        except_cond_as     ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
-                               POP_TOP STORE_FAST POP_TOP
+         except_cond1       ::= DUP_TOP expr COMPARE_OP jmp_false
+                                POP_TOP POP_TOP POP_TOP
+                                POP_EXCEPT
+         except_cond_as     ::= DUP_TOP expr COMPARE_OP POP_JUMP_IF_FALSE
+                                POP_TOP STORE_FAST POP_TOP
 
-        try_elsestmtl38    ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
-                               except_handler38 COME_FROM
-                               else_suitel opt_come_from_except
-        try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
-                               except_handler38
+         try_elsestmtl38    ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                except_handler38 COME_FROM
+                                else_suitel opt_come_from_except
+         try_except         ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                except_handler38
 
-        try_except38       ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
-                               except_handler38a
+         try_except38       ::= SETUP_FINALLY POP_BLOCK POP_TOP suite_stmts_opt
+                                except_handler38a
 
-        # suite_stmts has a return
-        try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
-                               except_handler38b
-        try_except38r      ::= SETUP_FINALLY return_except
-                               except_handler38b
-        return_except      ::= stmts POP_BLOCK return
+         # suite_stmts has a return
+         try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                                except_handler38b
+         try_except38r      ::= SETUP_FINALLY return_except
+                                except_handler38b
+         return_except      ::= stmts POP_BLOCK return
 
 
-        # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
-        # is a final return in the "except" block.
-        # So we treat the "return" separate from the other statements
-        cond_except_stmt      ::= except_cond1 except_stmts
-        cond_except_stmts_opt ::= cond_except_stmt*
+         # In 3.8 there seems to be some sort of code fiddle with POP_EXCEPT when there
+         # is a final return in the "except" block.
+         # So we treat the "return" separate from the other statements
+         cond_except_stmt      ::= except_cond1 except_stmts
+         cond_except_stmts_opt ::= cond_except_stmt*
 
-        try_except38r2     ::= SETUP_FINALLY
-                               suite_stmts_opt
-                               POP_BLOCK JUMP_FORWARD
-                               COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
-                               cond_except_stmts_opt
-                               POP_EXCEPT return
-                               END_FINALLY
-                               COME_FROM
+         try_except38r2     ::= SETUP_FINALLY
+                                suite_stmts_opt
+                                POP_BLOCK JUMP_FORWARD
+                                COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                                cond_except_stmts_opt
+                                POP_EXCEPT return
+                                END_FINALLY
+                                COME_FROM
 
-        try_except38r3     ::= SETUP_FINALLY
-                               suite_stmts_opt
-                               POP_BLOCK JUMP_FORWARD
-                               COME_FROM_FINALLY
-                               cond_except_stmts_opt
-                               POP_EXCEPT return
-                               COME_FROM
-                               END_FINALLY
-                               COME_FROM
+         try_except38r3     ::= SETUP_FINALLY
+                                suite_stmts_opt
+                                POP_BLOCK JUMP_FORWARD
+                                COME_FROM_FINALLY
+                                cond_except_stmts_opt
+                                POP_EXCEPT return
+                                COME_FROM
+                                END_FINALLY
+                                COME_FROM
 
 
-        try_except38r4     ::= SETUP_FINALLY
-                               returns_in_except
-                               COME_FROM_FINALLY
-                               except_cond1
-                               return
-                               COME_FROM
-                               END_FINALLY
+         try_except38r4     ::= SETUP_FINALLY
+                                returns_in_except
+                                COME_FROM_FINALLY
+                                except_cond1
+                                return
+                                COME_FROM
+                                END_FINALLY
 
 
-        # suite_stmts has a return
-        try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
-                               except_handler38b
-        try_except_as      ::= SETUP_FINALLY POP_BLOCK suite_stmts
-                               except_handler_as END_FINALLY COME_FROM
-        try_except_as      ::= SETUP_FINALLY suite_stmts
-                               except_handler_as END_FINALLY COME_FROM
+         # suite_stmts has a return
+         try_except38       ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                                except_handler38b
+         try_except_as      ::= SETUP_FINALLY POP_BLOCK suite_stmts
+                                except_handler_as END_FINALLY COME_FROM
+         try_except_as      ::= SETUP_FINALLY suite_stmts
+                                except_handler_as END_FINALLY COME_FROM
 
-        try_except_ret38   ::= SETUP_FINALLY returns except_ret38a
-        try_except_ret38a  ::= SETUP_FINALLY returns except_handler38c
-                               END_FINALLY come_from_opt
+         try_except_ret38   ::= SETUP_FINALLY returns except_ret38a
+         try_except_ret38a  ::= SETUP_FINALLY returns except_handler38c
+                                END_FINALLY come_from_opt
 
-        # Note: there is a suite_stmts_opt which seems
-        # to be bookkeeping which is not expressed in source code
-        except_ret38       ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
-                               CALL_FINALLY RETURN_VALUE COME_FROM
-                               COME_FROM_FINALLY
-                               suite_stmts_opt END_FINALLY
-        except_ret38a      ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
-                               expr ROT_FOUR
-                               POP_EXCEPT RETURN_VALUE END_FINALLY
+         # Note: there is a suite_stmts_opt which seems
+         # to be bookkeeping which is not expressed in source code
+         except_ret38       ::= SETUP_FINALLY expr ROT_FOUR POP_BLOCK POP_EXCEPT
+                                CALL_FINALLY RETURN_VALUE COME_FROM
+                                COME_FROM_FINALLY
+                                suite_stmts_opt END_FINALLY
+         except_ret38a      ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                                expr ROT_FOUR
+                                POP_EXCEPT RETURN_VALUE END_FINALLY
 
-        except_handler38   ::= _jump COME_FROM_FINALLY
-                               except_stmts END_FINALLY opt_come_from_except
-        except_handler38a  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
-                               POP_EXCEPT POP_TOP stmts END_FINALLY
+         except_handler38   ::= _jump COME_FROM_FINALLY
+                                except_stmts END_FINALLY opt_come_from_except
+         except_handler38a  ::= COME_FROM_FINALLY POP_TOP POP_TOP POP_TOP
+                                POP_EXCEPT POP_TOP stmts END_FINALLY
 
-        except_handler38c  ::= COME_FROM_FINALLY except_cond1a except_stmts
-                               POP_EXCEPT JUMP_FORWARD COME_FROM
-        except_handler_as  ::= COME_FROM_FINALLY except_cond_as tryfinallystmt
-                               POP_EXCEPT JUMP_FORWARD COME_FROM
+         except_handler38c  ::= COME_FROM_FINALLY except_cond1a except_stmts
+                                POP_EXCEPT JUMP_FORWARD COME_FROM
+         except_handler_as  ::= COME_FROM_FINALLY except_cond_as tryfinallystmt
+                                POP_EXCEPT JUMP_FORWARD COME_FROM
 
-        tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
-                               BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
-                               END_FINALLY
+         tryfinallystmt     ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                BEGIN_FINALLY COME_FROM_FINALLY suite_stmts_opt
+                                END_FINALLY
 
 
-        lc_setup_finally   ::= LOAD_CONST SETUP_FINALLY
-        call_finally_pt    ::= CALL_FINALLY POP_TOP
-        cf_cf_finally      ::= come_from_opt COME_FROM_FINALLY
-        pop_finally_pt     ::= POP_FINALLY POP_TOP
-        ss_end_finally     ::= suite_stmts END_FINALLY
-        sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
+         lc_setup_finally   ::= LOAD_CONST SETUP_FINALLY
+         call_finally_pt    ::= CALL_FINALLY POP_TOP
+         cf_cf_finally      ::= come_from_opt COME_FROM_FINALLY
+         pop_finally_pt     ::= POP_FINALLY POP_TOP
+         ss_end_finally     ::= suite_stmts END_FINALLY
+         sf_pb_call_returns ::= SETUP_FINALLY POP_BLOCK CALL_FINALLY returns
 
 
-        # FIXME: DRY rules below
-        tryfinally38rstmt  ::= sf_pb_call_returns
-                               cf_cf_finally
-                               ss_end_finally
-        tryfinally38rstmt  ::= sf_pb_call_returns
-                               cf_cf_finally END_FINALLY
-                               suite_stmts
-        tryfinally38rstmt  ::= sf_pb_call_returns
-                               cf_cf_finally POP_FINALLY
-                               ss_end_finally
-        tryfinally38rstmt  ::= sf_bp_call_returns
-                               COME_FROM_FINALLY POP_FINALLY
-                               ss_end_finally
+         # FIXME: DRY rules below
+         tryfinally38rstmt  ::= sf_pb_call_returns
+                                cf_cf_finally
+                                ss_end_finally
+         tryfinally38rstmt  ::= sf_pb_call_returns
+                                cf_cf_finally END_FINALLY
+                                suite_stmts
+         tryfinally38rstmt  ::= sf_pb_call_returns
+                                cf_cf_finally POP_FINALLY
+                                ss_end_finally
+         tryfinally38rstmt  ::= sf_bp_call_returns
+                                COME_FROM_FINALLY POP_FINALLY
+                                ss_end_finally
 
-        tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
-                               returns
-                               cf_cf_finally pop_finally_pt
-                               ss_end_finally POP_TOP
-        tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
-                               COME_FROM COME_FROM_FINALLY
-                               ss_end_finally
+         tryfinally38rstmt2 ::= lc_setup_finally POP_BLOCK call_finally_pt
+                                returns
+                                cf_cf_finally pop_finally_pt
+                                ss_end_finally POP_TOP
+         tryfinally38rstmt3 ::= SETUP_FINALLY expr POP_BLOCK CALL_FINALLY RETURN_VALUE
+                                COME_FROM COME_FROM_FINALLY
+                                ss_end_finally
 
-        tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
-                               BEGIN_FINALLY COME_FROM_FINALLY
-                               POP_FINALLY suite_stmts_opt END_FINALLY
+         tryfinally38stmt   ::= SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                BEGIN_FINALLY COME_FROM_FINALLY
+                                POP_FINALLY suite_stmts_opt END_FINALLY
 
-        tryfinally38astmt  ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
-                               BEGIN_FINALLY COME_FROM_FINALLY
-                               POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
+         tryfinally38astmt  ::= LOAD_CONST SETUP_FINALLY suite_stmts_opt POP_BLOCK
+                                BEGIN_FINALLY COME_FROM_FINALLY
+                                POP_FINALLY POP_TOP suite_stmts_opt END_FINALLY POP_TOP
         """
 
     def p_38walrus(self, args):
@@ -360,13 +365,24 @@ class Python38Parser(Python37Parser):
         """
         )
 
-    def customize_grammar_rules(self, tokens, customize):
-        super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+    def customize_reduce_checks_full38(self, tokens, customize):
+        """
+        Extra tests when a reduction is made in the full grammar.
+
+        Reductions here are extended from those used in the lambda grammar
+        """
         self.remove_rules_38()
+        self.check_reduce["pop_return"] = "tokens"
         self.check_reduce["whileTruestmt38"] = "tokens"
         self.check_reduce["whilestmt38"] = "tokens"
         self.check_reduce["try_elsestmtl38"] = "AST"
 
+        self.reduce_check_table["pop_return"] = pop_return_check
+
+    def customize_grammar_rules(self, tokens, customize):
+        super(Python37Parser, self).customize_grammar_rules(tokens, customize)
+        self.customize_reduce_checks_full38(tokens, customize)
+
         # For a rough break out on the first word. This may
         # include instructions that don't need customization,
         # but we'll do a finer check after the rough breakout.
@@ -421,11 +437,7 @@ class Python38Parser(Python37Parser):
         # Determine if we have an iteration CALL_FUNCTION_1.
         has_get_iter_call_function1 = False
         for i, token in enumerate(tokens):
-            if (
-                token == "GET_ITER"
-                and i < n - 2
-                and tokens[i + 1] == "CALL_FUNCTION_1"
-            ):
+            if token == "GET_ITER" and i < n - 2 and tokens[i + 1] == "CALL_FUNCTION_1":
                 has_get_iter_call_function1 = True
 
         for i, token in enumerate(tokens):
@@ -523,26 +535,26 @@ class Python38Parser(Python37Parser):
                 continue
 
             elif opname == "BUILD_STRING_2":
-               self.addRule(
-                   """
+                self.addRule(
+                    """
                      expr                  ::= formatted_value_debug
                      formatted_value_debug ::= LOAD_STR formatted_value2 BUILD_STRING_2
                      formatted_value_debug ::= LOAD_STR formatted_value1 BUILD_STRING_2
                    """,
-                   nop_func,
-               )
-               custom_ops_processed.add(opname)
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
 
             elif opname == "BUILD_STRING_3":
-               self.addRule(
-                   """
+                self.addRule(
+                    """
                      expr                  ::= formatted_value_debug
                      formatted_value_debug ::= LOAD_STR formatted_value2 LOAD_STR BUILD_STRING_3
                      formatted_value_debug ::= LOAD_STR formatted_value1 LOAD_STR BUILD_STRING_3
                    """,
-                   nop_func,
-               )
-               custom_ops_processed.add(opname)
+                    nop_func,
+                )
+                custom_ops_processed.add(opname)
 
             elif opname == "LOAD_CLOSURE":
                 self.addRule("""load_closure ::= LOAD_CLOSURE+""", nop_func)
@@ -574,15 +586,20 @@ class Python38Parser(Python37Parser):
                                     GET_ITER CALL_FUNCTION_1
                        """
                     self.addRule(rule, nop_func)
-
-
-
-
+            elif opname == "SETUP_WITH":
+                rules_str = """
+                  stmt        ::= with_as_pass
+                  with_as_pass ::= expr
+                                   SETUP_WITH store pass
+                                   POP_BLOCK BEGIN_FINALLY COME_FROM_WITH
+                                   with_suffix
+                 """
+                self.addRule(rules_str, nop_func)
 
     def reduce_is_invalid(self, rule, ast, tokens, first, last):
-        invalid = super(Python38Parser,
-                        self).reduce_is_invalid(rule, ast,
-                                                tokens, first, last)
+        invalid = super(Python38Parser, self).reduce_is_invalid(
+            rule, ast, tokens, first, last
+        )
         self.remove_rules_38()
         if invalid:
             return invalid
@@ -610,7 +627,7 @@ if __name__ == "__main__":
     p = Python38Parser()
     p.remove_rules_38()
     p.check_grammar()
-    from xdis.version_info import PYTHON_VERSION_TRIPLE, IS_PYPY
+    from xdis.version_info import IS_PYPY, PYTHON_VERSION_TRIPLE
 
     if PYTHON_VERSION_TRIPLE[:2] == (3, 8):
         lhs, rhs, tokens, right_recursive, dup_rhs = p.check_sets()
@@ -633,7 +650,9 @@ if __name__ == "__main__":
         remain_tokens = set(remain_tokens) - opcode_set
         print(remain_tokens)
         import sys
+
         if len(sys.argv) > 1:
             from spark_parser.spark import rule2str
+
             for rule in sorted(p.rule2name.items()):
                 print(rule2str(rule[0]))

uncompyle6/parsers/reducecheck/and_check.py

@@ -5,9 +5,7 @@
 
 
 
-def and_invalid(
-    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
- ) -> bool:
+def and_invalid( self, lhs, n, rule, ast, tokens, first, last):
     jmp = ast[1]
     if jmp.kind.startswith("jmp_"):
         if last == n:

uncompyle6/parsers/reducecheck/and_not_check.py

@@ -3,7 +3,7 @@
 
 def and_not_check(
     self, lhs, n, rule, ast, tokens, first, last
-) -> bool:
+):
     jmp = ast[1]
     if jmp.kind.startswith("jmp_"):
         if last == n:

uncompyle6/parsers/reducecheck/for_block_check.py

@@ -1,10 +1,9 @@
-#  Copyright (c) 2022 Rocky Bernstein
+#  Copyright (c) 2022, 2024 Rocky Bernstein
 
 from uncompyle6.scanners.tok import Token
 
 
-def for_block_invalid(self, lhs, n, rule, tree, tokens, first: int, last: int) -> bool:
-
+def for_block_invalid(self, lhs, n, rule, tree, tokens, first, last):
     # print("XXX", first, last)
     # for t in range(first, last):
     #     print(tokens[t])
@@ -51,8 +50,8 @@ def for_block_invalid(self, lhs, n, rule, tree, tokens, first: int, last: int) -
         pop_jump_index -= 1
 
     # FIXME: something is fishy when and EXTENDED ARG is needed before the
-    # pop_jump_index instruction to get the argment. In this case, the
-    # _ifsmtst_jump can jump to a spot beyond the come_froms.
+    # pop_jump_index instruction to get the argument. In this case, the
+    # _ifsmtst_jump can jump to a spot beyond the ``come_froms``.
     # That is going on in the non-EXTENDED_ARG case is that the POP_JUMP_IF
     # jumps to a JUMP_(FORWARD) which is changed into an EXTENDED_ARG POP_JUMP_IF
     # to the jumped forwarded address

uncompyle6/parsers/reducecheck/ifelsestmt2.py

@@ -44,7 +44,6 @@ IFELSE_STMT_RULES = frozenset(
 
 
 def ifelsestmt2(self, lhs, n, rule, tree, tokens, first, last):
-
     if (last + 1) < n and tokens[last + 1] == "COME_FROM_LOOP" and lhs != "ifelsestmtc":
         # ifelsestmt jumped outside of loop. No good.
         return True
@@ -66,7 +65,7 @@ def ifelsestmt2(self, lhs, n, rule, tree, tokens, first, last):
         if raise_stmt1 == "raise_stmt1" and raise_stmt1[0] in ("LOAD_ASSERT",):
             return True
 
-    # Make sure all of the "come froms" offset at the
+    # Make sure all of the "come_froms" offset at the
     # end of the "if" come from somewhere inside the "if".
     # Since the come_froms are ordered so that lowest
     # offset COME_FROM is last, it is sufficient to test

uncompyle6/parsers/reducecheck/ifstmt.py

@@ -1,13 +1,16 @@
-#  Copyright (c) 2020 Rocky Bernstein
+#  Copyright (c) 2020, 2023 Rocky Bernstein
 
 
 def ifstmt(self, lhs, n, rule, ast, tokens, first, last):
+
+    first_offset = tokens[first].off2int(prefer_last=False)
+
     if lhs == "ifstmtl":
         if last == n:
             last -= 1
             pass
         if tokens[last].attr and isinstance(tokens[last].attr, int):
-            if tokens[first].offset >= tokens[last].attr:
+            if first_offset >= tokens[last].attr:
                 return True
             pass
         pass
@@ -36,7 +39,7 @@ def ifstmt(self, lhs, n, rule, ast, tokens, first, last):
                 if tokens[l] == "JUMP_FORWARD":
                     return tokens[l].attr != pjif_target
                 return True
-            elif lhs == "ifstmtl" and tokens[first].off2int() > pjif_target:
+            elif lhs == "ifstmtl" and first_offset > pjif_target:
                 # A conditional JUMP to the loop is expected for "ifstmtl"
                 return False
             pass
@@ -55,7 +58,7 @@ def ifstmt(self, lhs, n, rule, ast, tokens, first, last):
             if len(test) > 1 and test[1].kind.startswith("jmp_"):
                 jmp_target = test[1][0].attr
                 if (
-                    tokens[first].off2int(prefer_last=True)
+                    first_offset
                     <= jmp_target
                     < tokens[last].off2int(prefer_last=False)
                 ):

uncompyle6/parsers/reducecheck/ifstmts_jump.py

@@ -4,7 +4,6 @@ from uncompyle6.scanners.tok import Token
 
 
 def ifstmts_jump(self, lhs, n, rule, ast, tokens, first, last):
-
     if len(rule[1]) <= 1 or not ast:
         return False
 
@@ -24,7 +23,7 @@ def ifstmts_jump(self, lhs, n, rule, ast, tokens, first, last):
         pop_jump_index -= 1
 
     # FIXME: something is fishy when and EXTENDED ARG is needed before the
-    # pop_jump_index instruction to get the argment. In this case, the
+    # pop_jump_index instruction to get the argument. In this case, the
     # _ifsmtst_jump can jump to a spot beyond the come_froms.
     # That is going on in the non-EXTENDED_ARG case is that the POP_JUMP_IF
     # jumps to a JUMP_(FORWARD) which is changed into an EXTENDED_ARG POP_JUMP_IF
@@ -34,16 +33,11 @@ def ifstmts_jump(self, lhs, n, rule, ast, tokens, first, last):
 
     pop_jump_offset = tokens[pop_jump_index].off2int(prefer_last=False)
     if isinstance(come_froms, Token):
-        if (
-            tokens[pop_jump_index].attr < pop_jump_offset and ast[0] != "pass"
-        ):
+        if tokens[pop_jump_index].attr < pop_jump_offset and ast[0] != "pass":
             # This is a jump backwards to a loop. All bets are off here when there the
             # unless statement is "pass" which has no instructions associated with it.
             return False
-        return (
-            come_froms.attr is not None
-            and pop_jump_offset > come_froms.attr
-        )
+        return come_froms.attr is not None and pop_jump_offset > come_froms.attr
 
     elif len(come_froms) == 0:
         return False

uncompyle6/parsers/reducecheck/joined_str_check.py

@@ -13,9 +13,7 @@
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-def joined_str_invalid(
-    self, lhs: str, n: int, rule, tree, tokens: list, first: int, last: int
-) -> bool:
+def joined_str_invalid(self, lhs, n, rule, tree, tokens, first, last):
     # In Python 3.8, there is a new "=" specifier.
     # See https://docs.python.org/3/whatsnew/3.8.html#f-strings-support-for-self-documenting-expressions-and-debugging
     # We detect this here inside joined_str by looking for an

uncompyle6/parsers/reducecheck/pop_return.py

@@ -0,0 +1,10 @@
+#  Copyright (c) 2020 Rocky Bernstein
+
+
+def pop_return_check(
+    self, lhs: str, n: int, rule, ast, tokens: list, first: int, last: int
+) -> bool:
+    # If the first instruction of return_expr (the instruction after POP_TOP) is
+    # has a linestart, then the POP_TOP was probably part of the previous
+    # statement, such as a call() where the return value is discarded.
+    return tokens[first + 1].linestart

uncompyle6/parsers/treenode.py

@@ -1,9 +1,6 @@
-import sys
 from uncompyle6.scanners.tok import NoneToken
 from spark_parser.ast import AST as spark_AST
 
-intern = sys.intern
-
 
 class SyntaxTree(spark_AST):
     def __init__(self, *args, transformed_by=None, **kwargs):

uncompyle6/scanner.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016, 2018-2022 by Rocky Bernstein
+#  Copyright (c) 2016, 2018-2024 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #  Copyright (c) 1999 John Aycock
@@ -21,12 +21,10 @@ scanner/ingestion module. From here we call various version-specific
 scanners, e.g. for Python 2.7 or 3.4.
 """
 
-from typing import Optional, Tuple
+import sys
 from array import array
 from collections import namedtuple
 
-from uncompyle6.scanners.tok import Token
-from xdis.version_info import IS_PYPY, version_tuple_to_str
 import xdis
 from xdis import (
     Bytecode,
@@ -36,6 +34,9 @@ from xdis import (
     instruction_size,
     next_offset,
 )
+from xdis.version_info import IS_PYPY, version_tuple_to_str
+
+from uncompyle6.scanners.tok import Token
 
 # The byte code versions we support.
 # Note: these all have to be tuples of 2 ints
@@ -77,8 +78,10 @@ CANONIC2VERSION["3.5.2"] = 3.5
 
 
 # FIXME: DRY
+intern = sys.intern
 L65536 = 65536
 
+
 def long(num):
     return num
 
@@ -86,7 +89,7 @@ def long(num):
 CONST_COLLECTIONS = ("CONST_LIST", "CONST_SET", "CONST_DICT", "CONST_MAP")
 
 
-class Code(object):
+class Code:
     """
     Class for representing code-objects.
 
@@ -95,23 +98,29 @@ class Code(object):
     """
 
     def __init__(self, co, scanner, classname=None, show_asm=None):
+        # Full initialization is given below, but for linters
+        # well set up some initial values.
+        self.co_code = None  # Really either bytes for >= 3.0 and string in < 3.0
+
         for i in dir(co):
             if i.startswith("co_"):
                 setattr(self, i, getattr(co, i))
         self._tokens, self._customize = scanner.ingest(co, classname, show_asm=show_asm)
 
 
-class Scanner(object):
+class Scanner:
     def __init__(self, version: tuple, show_asm=None, is_pypy=False):
         self.version = version
         self.show_asm = show_asm
         self.is_pypy = is_pypy
 
         if version[:2] in PYTHON_VERSIONS:
-            v_str = f"""opcode_{version_tuple_to_str(version, start=0, end=2, delimiter="")}"""
+            v_str = "opcode_%s" % version_tuple_to_str(
+                version, start=0, end=2, delimiter=""
+            )
             if is_pypy:
                 v_str += "pypy"
-            exec(f"""from xdis.opcodes import {v_str}""")
+            exec("from xdis.opcodes import %s" % v_str)
             exec("self.opc = %s" % v_str)
         else:
             raise TypeError(
@@ -124,9 +133,7 @@ class Scanner(object):
         # FIXME: This weird Python2 behavior is not Python3
         self.resetTokenClass()
 
-    def bound_collection_from_tokens(
-        self, tokens, t, i, collection_type
-    ):
+    def bound_collection_from_tokens(self, tokens, t, i, collection_type):
         count = t.attr
         assert isinstance(count, int)
 
@@ -171,11 +178,11 @@ class Scanner(object):
                 has_extended_arg=False,
             )
         )
-        if tokens[j] == "LOAD_CONST":
-            opname = "ADD_VALUE"
-        else:
-            opname = "ADD_VALUE_VAR"
         for j in range(collection_start, i):
+            if tokens[j] == "LOAD_CONST":
+                opname = "ADD_VALUE"
+            else:
+                opname = "ADD_VALUE_VAR"
             new_tokens.append(
                 Token(
                     opname=opname,
@@ -272,7 +279,7 @@ class Scanner(object):
             for _ in range(instruction_size(op, self.opc)):
                 self.prev_op.append(offset)
 
-    def is_jump_forward(self, offset: int) -> bool:
+    def is_jump_forward(self, offset):
         """
         Return True if the code at offset is some sort of jump forward.
         That is, it is ether "JUMP_FORWARD" or an absolute jump that
@@ -285,10 +292,16 @@ class Scanner(object):
             return False
         return offset < self.get_target(offset)
 
+    def ingest(self, co, classname=None, code_objects={}, show_asm=None):
+        """
+        Code to tokenize disassembly. Subclasses must implement this.
+        """
+        raise NotImplementedError("This method should have been implemented")
+
     def prev_offset(self, offset: int) -> int:
         return self.insts[self.offset2inst_index[offset] - 1].offset
 
-    def get_inst(self, offset: int):
+    def get_inst(self, offset):
         # Instructions can get moved as a result of EXTENDED_ARGS removal.
         # So if "offset" is not in self.offset2inst_index, then
         # we assume that it was an instruction moved back.
@@ -299,7 +312,7 @@ class Scanner(object):
             assert self.code[offset] == self.opc.EXTENDED_ARG
         return self.insts[self.offset2inst_index[offset]]
 
-    def get_target(self, offset: int, extended_arg: int = 0) -> int:
+    def get_target(self, offset, extended_arg=0):
         """
         Get next instruction offset for op located at given <offset>.
         NOTE: extended_arg is no longer used
@@ -312,11 +325,11 @@ class Scanner(object):
             target = next_offset(inst.opcode, self.opc, inst.offset)
         return target
 
-    def get_argument(self, pos: int):
+    def get_argument(self, pos):
         arg = self.code[pos + 1] + self.code[pos + 2] * 256
         return arg
 
-    def next_offset(self, op, offset: int) -> int:
+    def next_offset(self, op, offset):
         return xdis.next_offset(op, self.opc, offset)
 
     def print_bytecode(self):
@@ -328,7 +341,7 @@ class Scanner(object):
             else:
                 print("%i\t%s\t" % (i, self.opname[op]))
 
-    def first_instr(self, start: int, end: int, instr, target=None, exact=True):
+    def first_instr(self, start, end, instr, target=None, exact=True):
         """
         Find the first <instr> in the block from start to end.
         <instr> is any python bytecode instruction or a list of opcodes
@@ -362,9 +375,7 @@ class Scanner(object):
                         result_offset = offset
         return result_offset
 
-    def last_instr(
-        self, start: int, end: int, instr, target=None, exact=True
-    ) -> Optional[int]:
+    def last_instr(self, start, end, instr, target=None, exact=True):
         """
         Find the last <instr> in the block from start to end.
         <instr> is any python bytecode instruction or a list of opcodes
@@ -429,7 +440,7 @@ class Scanner(object):
         """
         try:
             None in instr
-        except:
+        except Exception:
             instr = [instr]
 
         first = self.offset2inst_index[start]
@@ -460,9 +471,7 @@ class Scanner(object):
     # FIXME: this is broken on 3.6+. Replace remaining (2.x-based) calls
     # with inst_matches
 
-    def all_instr(
-        self, start: int, end: int, instr, target=None, include_beyond_target=False
-    ):
+    def all_instr(self, start, end, instr, target=None, include_beyond_target=False):
         """
         Find all `instr` in the block from start to end.
         `instr` is any Python opcode or a list of opcodes
@@ -483,7 +492,6 @@ class Scanner(object):
         result = []
         extended_arg = 0
         for offset in self.op_range(start, end):
-
             op = code[offset]
 
             if op == self.opc.EXTENDED_ARG:
@@ -542,7 +550,6 @@ class Scanner(object):
                 offset = inst.offset
                 continue
             if last_was_extarg:
-
                 # j = self.stmts.index(inst.offset)
                 # self.lines[j] = offset
 
@@ -589,40 +596,19 @@ class Scanner(object):
     def resetTokenClass(self):
         return self.setTokenClass(Token)
 
-    def restrict_to_parent(self, target: int, parent) -> int:
+    def restrict_to_parent(self, target, parent):
         """Restrict target to parent structure boundaries."""
         if not (parent["start"] < target < parent["end"]):
             target = parent["end"]
         return target
 
-    def setTokenClass(self, tokenClass) -> Token:
+    def setTokenClass(self, tokenClass):
+        # assert isinstance(tokenClass, types.ClassType)
         self.Token = tokenClass
         return self.Token
 
 
-# TODO: after the next xdis release, use from there instead.
-def parse_fn_counts_30_35(argc: int) -> Tuple[int, int, int]:
-    """
-    In Python 3.0 to 3.5 MAKE_CLOSURE and MAKE_FUNCTION encode
-    arguments counts of positional, default + named, and annotation
-    arguments a particular kind of encoding where each of
-    the entry a a packe byted value of the lower 24 bits
-    of ``argc``.  The high bits of argc may have come from
-    an EXTENDED_ARG instruction. Here, we unpack the values
-    from the ``argc`` int and return a triple of the
-    positional args, named_args, and annotation args.
-    """
-    annotate_count = (argc >> 16) & 0x7FFF
-    # For some reason that I don't understand, annotate_args is off by one
-    # when there is an EXENDED_ARG instruction from what is documented in
-    # https://docs.python.org/3.4/library/dis.html#opcode-MAKE_CLOSURE
-    if annotate_count > 1:
-        annotate_count -= 1
-    return ((argc & 0xFF), (argc >> 8) & 0xFF, annotate_count)
-
-
 def get_scanner(version, is_pypy=False, show_asm=None):
-
     # If version is a string, turn that into the corresponding float.
     if isinstance(version, str):
         if version not in canonic_python_version:
@@ -670,7 +656,8 @@ def get_scanner(version, is_pypy=False, show_asm=None):
             )
     else:
         raise RuntimeError(
-            f"Unsupported Python version, {version_tuple_to_str(version)}, for decompilation"
+            "Unsupported Python version, %s, for decompilation"
+            % version_tuple_to_str(version)
         )
     return scanner
 
@@ -682,5 +669,6 @@ if __name__ == "__main__":
     # scanner = get_scanner('2.7.13', True)
     # scanner = get_scanner(sys.version[:5], False)
     from xdis.version_info import PYTHON_VERSION_TRIPLE
+
     scanner = get_scanner(PYTHON_VERSION_TRIPLE, IS_PYPY, True)
     tokens, customize = scanner.ingest(co, {}, show_asm="after")

uncompyle6/scanners/scanner15.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2016-2018, 2021-2022 by Rocky Bernstein
+#  Copyright (c) 2016-2018, 2021-2023 by Rocky Bernstein
 """
 Python 1.5 bytecode decompiler massaging.
 
@@ -7,12 +7,15 @@ grammar parsing.
 """
 
 import uncompyle6.scanners.scanner21 as scan
+
 # from uncompyle6.scanners.scanner26 import ingest as  ingest26
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis.opcodes import opcode_15
+
 JUMP_OPS = opcode_15.JUMP_OPS
 
+
 # We base this off of 2.2 instead of the other way around
 # because we cleaned things up this way.
 # The history is that 2.7 support is the cleanest,
@@ -23,7 +26,7 @@ class Scanner15(scan.Scanner21):
         self.opc = opcode_15
         self.opname = opcode_15.opname
         self.version = (1, 5)
-        self.genexpr_name = '<generator expression>'
+        self.genexpr_name = "<generator expression>"
         return
 
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
@@ -36,18 +39,22 @@ class Scanner15(scan.Scanner21):
         Some transformations are made to assist the deparsing grammar:
            -  various types of LOAD_CONST's are categorized in terms of what they load
            -  COME_FROM instructions are added to assist parsing control structures
-           -  operands with stack argument counts or flag masks are appended to the opcode name, e.g.:
+           -  operands with stack argument counts or flag masks are appended to the
+              opcode name, e.g.:
               *  BUILD_LIST, BUILD_SET
-              *  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
+              *  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional
+                 arguments
            -  EXTENDED_ARGS instructions are removed
 
-        Also, when we encounter certain tokens, we add them to a set which will cause custom
-        grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
-        cause specific rules for the specific number of arguments they take.
+        Also, when we encounter certain tokens, we add them to a set which will cause
+        custom grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or
+        BUILD_LIST cause specific rules for the specific number of arguments they take.
         """
-        tokens, customize = scan.Scanner21.ingest(self, co, classname, code_objects, show_asm)
+        tokens, customize = scan.Scanner21.ingest(
+            self, co, classname, code_objects, show_asm
+        )
         for t in tokens:
             if t.op == self.opc.UNPACK_LIST:
-                t.kind = 'UNPACK_LIST_%d' % t.attr
+                t.kind = "UNPACK_LIST_%d" % t.attr
             pass
         return tokens, customize

uncompyle6/scanners/scanner2.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2023 by Rocky Bernstein
+#  Copyright (c) 2015-2024 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
@@ -36,13 +36,13 @@ Finally we save token information.
 from __future__ import print_function
 
 from copy import copy
-
-from xdis import code2num, iscode, op_has_argument, instruction_size
-from xdis.bytecode import _get_const_info, _get_name_info
-from uncompyle6.scanner import Scanner, Token
-
 from sys import intern
 
+from xdis import code2num, instruction_size, iscode, op_has_argument
+from xdis.bytecode import _get_const_info
+
+from uncompyle6.scanner import Scanner, Token
+
 
 class Scanner2(Scanner):
     def __init__(self, version, show_asm=None, is_pypy=False):
@@ -55,7 +55,7 @@ class Scanner2(Scanner):
         self.load_asserts = set([])
 
         # Create opcode classification sets
-        # Note: super initilization above initializes self.opc
+        # Note: super initialization above initializes self.opc
 
         # Ops that start SETUP_ ... We will COME_FROM with these names
         # Some blocks and END_ statements. And they can start
@@ -200,15 +200,23 @@ class Scanner2(Scanner):
         grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
         cause specific rules for the specific number of arguments they take.
         """
+
         if not show_asm:
             show_asm = self.show_asm
 
         bytecode = self.build_instructions(co)
 
-        # show_asm = 'after'
         if show_asm in ("both", "before"):
-            for instr in bytecode.get_instructions(co):
-                print(instr.disassemble())
+            print("\n# ---- disassembly:")
+            bytecode.disassemble_bytes(
+                co.co_code,
+                varnames=co.co_varnames,
+                names=co.co_names,
+                constants=co.co_consts,
+                cells=bytecode._cell_names,
+                linestarts=bytecode._linestarts,
+                asm_format="extended",
+            )
 
         # list of tokens/instructions
         new_tokens = []
@@ -228,7 +236,6 @@ class Scanner2(Scanner):
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
         for i in self.op_range(0, codelen):
-
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
@@ -320,7 +327,14 @@ class Scanner2(Scanner):
                     "BUILD_SET",
                 ):
                     t = Token(
-                        op_name, oparg, pattr, offset, self.linestarts.get(offset, None), op, has_arg, self.opc
+                        op_name,
+                        oparg,
+                        pattr,
+                        offset,
+                        self.linestarts.get(offset, None),
+                        op,
+                        has_arg,
+                        self.opc,
                     )
                     collection_type = op_name.split("_")[1]
                     next_tokens = self.bound_collection_from_tokens(
@@ -360,7 +374,6 @@ class Scanner2(Scanner):
                         pattr = const
                         pass
                 elif op in self.opc.NAME_OPS:
-                    _, pattr = _get_name_info(oparg, names)
                     pattr = names[oparg]
                 elif op in self.opc.JREL_OPS:
                     #  use instead: hasattr(self, 'patch_continue'): ?
@@ -422,7 +435,7 @@ class Scanner2(Scanner):
 
                     # EXTENDED_ARG doesn't appear in instructions,
                     # but is instead the next opcode folded into it, and has the offset
-                    # of the EXTENDED_ARG. Therefor in self.offset2nist_index we'll find
+                    # of the EXTENDED_ARG. Therefore in self.offset2nist_index we'll find
                     # the instruction at the previous EXTENDED_ARG offset which is 3
                     # bytes back.
                     if j is None and offset > self.opc.ARG_MAX_VALUE:
@@ -482,6 +495,7 @@ class Scanner2(Scanner):
             pass
 
         if show_asm in ("both", "after"):
+            print("\n# ---- tokenization:")
             for t in new_tokens:
                 print(t.format(line_prefix=""))
             print()
@@ -531,14 +545,17 @@ class Scanner2(Scanner):
         for s in stmt_list:
             if code[s] == self.opc.JUMP_ABSOLUTE and s not in pass_stmts:
                 target = self.get_target(s)
-                if target > s or (self.lines and self.lines[last_stmt].l_no == self.lines[s].l_no):
+                if target > s or (
+                    self.lines and self.lines[last_stmt].l_no == self.lines[s].l_no
+                ):
                     stmts.remove(s)
                     continue
                 j = self.prev[s]
                 while code[j] == self.opc.JUMP_ABSOLUTE:
                     j = self.prev[j]
                 if (
-                    self.version >= (2, 3) and self.opname_for_offset(j) == "LIST_APPEND"
+                    self.version >= (2, 3)
+                    and self.opname_for_offset(j) == "LIST_APPEND"
                 ):  # list comprehension
                     stmts.remove(s)
                     continue
@@ -915,8 +932,7 @@ class Scanner2(Scanner):
 
             # Is it an "and" inside an "if" or "while" block
             if op == self.opc.PJIF:
-
-                # Search for other POP_JUMP_IF_...'s targetting the
+                # Search for other POP_JUMP_IF_...'s targeting the
                 # same target, of the current POP_JUMP_... instruction,
                 # starting from current offset, and filter everything inside inner 'or'
                 # jumps and mid-line ifs
@@ -1015,7 +1031,7 @@ class Scanner2(Scanner):
                         ):
                             self.fixed_jumps[offset] = rtarget
                         else:
-                            # note test for < 2.7 might be superflous although informative
+                            # note test for < 2.7 might be superfluous although informative
                             # for 2.7 a different branch is taken and the below code is handled
                             # under: elif op in self.pop_jump_if_or_pop
                             # below
@@ -1105,9 +1121,8 @@ class Scanner2(Scanner):
             if code_pre_rtarget in self.jump_forward:
                 if_end = self.get_target(pre_rtarget)
 
-                # Is this a loop and not an "if" statment?
+                # Is this a loop and not an "if" statement?
                 if (if_end < pre_rtarget) and (pre[if_end] in self.setup_loop_targets):
-
                     if if_end > start:
                         return
                     else:
@@ -1328,9 +1343,9 @@ class Scanner2(Scanner):
 
                 # FIXME FIXME FIXME
                 # All the conditions are horrible, and I am not sure I
-                # undestand fully what's going l
+                # understand fully what's going l
                 # We REALLY REALLY  need a better way to handle control flow
-                # Expecially for < 2.7
+                # Especially for < 2.7
                 if label is not None and label != -1:
                     if self.version[:2] == (2, 7):
                         # FIXME: rocky: I think we need something like this...
@@ -1453,3 +1468,16 @@ class Scanner2(Scanner):
             instr_offsets = filtered
             filtered = []
         return instr_offsets
+
+
+if __name__ == "__main__":
+    import inspect
+
+    from xdis.version_info import PYTHON_VERSION_TRIPLE
+
+    co = inspect.currentframe().f_code
+
+    tokens, customize = Scanner2(PYTHON_VERSION_TRIPLE).ingest(co)
+    for t in tokens:
+        print(t)
+    pass

uncompyle6/scanners/scanner26.py

@@ -1,4 +1,4 @@
-#  Copyright (c) 2015-2017, 2021-2022 by Rocky Bernstein
+#  Copyright (c) 2015-2017, 2021-2022, 2024 by Rocky Bernstein
 #  Copyright (c) 2005 by Dan Pascu <dan@windowmaker.org>
 #  Copyright (c) 2000-2002 by hartmut Goebel <h.goebel@crazy-compilers.com>
 #
@@ -23,33 +23,36 @@ use in deparsing.
 """
 
 import sys
-import uncompyle6.scanners.scanner2 as scan
 
 # bytecode verification, verify(), uses JUMP_OPs from here
 from xdis import iscode
-from xdis.opcodes import opcode_26
 from xdis.bytecode import _get_const_info
+from xdis.opcodes import opcode_26
 
+import uncompyle6.scanners.scanner2 as scan
 from uncompyle6.scanner import Token
 
 intern = sys.intern
 
 JUMP_OPS = opcode_26.JUMP_OPS
 
+
 class Scanner26(scan.Scanner2):
     def __init__(self, show_asm=False):
         super(Scanner26, self).__init__((2, 6), show_asm)
 
         # "setup" opcodes
-        self.setup_ops = frozenset([
-            self.opc.SETUP_EXCEPT, self.opc.SETUP_FINALLY,
-            ])
+        self.setup_ops = frozenset(
+            [
+                self.opc.SETUP_EXCEPT,
+                self.opc.SETUP_FINALLY,
+            ]
+        )
 
         return
 
     def ingest(self, co, classname=None, code_objects={}, show_asm=None):
-        """
-        Create "tokens" the bytecode of an Python code object. Largely these
+        """Create "tokens" the bytecode of an Python code object. Largely these
         are the opcode name, but in some cases that has been modified to make parsing
         easier.
         returning a list of uncompyle6 Token's.
@@ -57,14 +60,17 @@ class Scanner26(scan.Scanner2):
         Some transformations are made to assist the deparsing grammar:
            -  various types of LOAD_CONST's are categorized in terms of what they load
            -  COME_FROM instructions are added to assist parsing control structures
-           -  operands with stack argument counts or flag masks are appended to the opcode name, e.g.:
+           -  operands with stack argument counts or flag masks are appended to the
+              opcode name, e.g.:
               *  BUILD_LIST, BUILD_SET
-              *  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional arguments
+              *  MAKE_FUNCTION and FUNCTION_CALLS append the number of positional
+                 arguments
            -  EXTENDED_ARGS instructions are removed
 
-        Also, when we encounter certain tokens, we add them to a set which will cause custom
-        grammar rules. Specifically, variable arg tokens like MAKE_FUNCTION or BUILD_LIST
-        cause specific rules for the specific number of arguments they take.
+        Also, when we encounter certain tokens, we add them to a set
+        which will cause custom grammar rules. Specifically, variable
+        arg tokens like MAKE_FUNCTION or BUILD_LIST cause specific
+        rules for the specific number of arguments they take.
         """
 
         if not show_asm:
@@ -74,8 +80,9 @@ class Scanner26(scan.Scanner2):
 
         # show_asm = 'after'
         if show_asm in ("both", "before"):
+            print("\n# ---- disassembly:")
             for instr in bytecode.get_instructions(co):
-                print(instr.disassemble())
+                print(instr.disassemble(self.opc))
 
         # Container for tokens
         tokens = []
@@ -94,17 +101,18 @@ class Scanner26(scan.Scanner2):
         # 'LOAD_ASSERT' is used in assert statements.
         self.load_asserts = set()
         for i in self.op_range(0, codelen):
-
             # We need to detect the difference between:
             #   raise AssertionError
             #  and
             #   assert ...
-            if (self.code[i] == self.opc.JUMP_IF_TRUE and
-                i + 4 < codelen and
-                self.code[i+3] == self.opc.POP_TOP and
-                self.code[i+4] == self.opc.LOAD_GLOBAL):
-                if names[self.get_argument(i+4)] == 'AssertionError':
-                    self.load_asserts.add(i+4)
+            if (
+                self.code[i] == self.opc.JUMP_IF_TRUE
+                and i + 4 < codelen
+                and self.code[i + 3] == self.opc.POP_TOP
+                and self.code[i + 4] == self.opc.LOAD_GLOBAL
+            ):
+                if names[self.get_argument(i + 4)] == "AssertionError":
+                    self.load_asserts.add(i + 4)
 
         jump_targets = self.find_jump_targets(show_asm)
         # contains (code, [addrRefToCode])
@@ -129,7 +137,8 @@ class Scanner26(scan.Scanner2):
             i += 1
             op = self.code[offset]
             op_name = self.opname[op]
-            oparg = None; pattr = None
+            oparg = None
+            pattr = None
 
             if offset in jump_targets:
                 jump_idx = 0
@@ -140,28 +149,37 @@ class Scanner26(scan.Scanner2):
                 # properly. For example, a "loop" with an "if" nested in it should have the
                 # "loop" tag last so the grammar rule matches that properly.
                 last_jump_offset = -1
-                for jump_offset  in sorted(jump_targets[offset], reverse=True):
+                for jump_offset in sorted(jump_targets[offset], reverse=True):
                     if jump_offset != last_jump_offset:
-                        tokens.append(Token(
-                            'COME_FROM', jump_offset, repr(jump_offset),
-                            offset="%s_%d" % (offset, jump_idx),
-                            has_arg = True))
+                        tokens.append(
+                            Token(
+                                "COME_FROM",
+                                jump_offset,
+                                repr(jump_offset),
+                                offset="%s_%d" % (offset, jump_idx),
+                                has_arg=True,
+                            )
+                        )
                         jump_idx += 1
                         last_jump_offset = jump_offset
             elif offset in self.thens:
-                tokens.append(Token(
-                    'THEN', None, self.thens[offset],
-                    offset="%s_0" % offset,
-                    has_arg = True))
+                tokens.append(
+                    Token(
+                        "THEN",
+                        None,
+                        self.thens[offset],
+                        offset="%s_0" % offset,
+                        has_arg=True,
+                    )
+                )
 
-            has_arg = (op >= self.opc.HAVE_ARGUMENT)
+            has_arg = op >= self.opc.HAVE_ARGUMENT
             if has_arg:
                 oparg = self.get_argument(offset) + extended_arg
                 extended_arg = 0
                 if op == self.opc.EXTENDED_ARG:
-                     extended_arg += self.extended_arg_val(oparg)
-                     continue
-
+                    extended_arg += self.extended_arg_val(oparg)
+                    continue
 
                 # Note: name used to match on rather than op since
                 # BUILD_SET isn't in earlier Pythons.
@@ -170,7 +188,14 @@ class Scanner26(scan.Scanner2):
                     "BUILD_SET",
                 ):
                     t = Token(
-                        op_name, oparg, pattr, offset, self.linestarts.get(offset, None), op, has_arg, self.opc
+                        op_name,
+                        oparg,
+                        pattr,
+                        offset,
+                        self.linestarts.get(offset, None),
+                        op,
+                        has_arg,
+                        self.opc,
                     )
 
                     collection_type = op_name.split("_")[1]
@@ -219,8 +244,8 @@ class Scanner26(scan.Scanner2):
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.
-                        if len(tokens) and tokens[-1].kind == 'JUMP_BACK':
-                            tokens[-1].kind = intern('CONTINUE')
+                        if len(tokens) and tokens[-1].kind == "JUMP_BACK":
+                            tokens[-1].kind = intern("CONTINUE")
 
                 elif op in self.opc.JABS_OPS:
                     pattr = repr(oparg)
@@ -238,17 +263,23 @@ class Scanner26(scan.Scanner2):
                 # CE - Hack for >= 2.5
                 #      Now all values loaded via LOAD_CLOSURE are packed into
                 #      a tuple before calling MAKE_CLOSURE.
-                if (self.version >= (2, 5) and op == self.opc.BUILD_TUPLE and
-                    self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE):
+                if (
+                    self.version >= (2, 5)
+                    and op == self.opc.BUILD_TUPLE
+                    and self.code[self.prev[offset]] == self.opc.LOAD_CLOSURE
+                ):
                     continue
                 else:
-                    op_name = '%s_%d' % (op_name, oparg)
+                    op_name = "%s_%d" % (op_name, oparg)
                     customize[op_name] = oparg
             elif self.version > (2, 0) and op == self.opc.CONTINUE_LOOP:
                 customize[op_name] = 0
-            elif op_name in """
+            elif (
+                op_name
+                in """
                  CONTINUE_LOOP EXEC_STMT LOAD_LISTCOMP LOAD_SETCOMP
-                  """.split():
+                  """.split()
+            ):
                 customize[op_name] = 0
             elif op == self.opc.JUMP_ABSOLUTE:
                 # Further classify JUMP_ABSOLUTE into backward jumps
@@ -264,23 +295,24 @@ class Scanner26(scan.Scanner2):
                 # rule for that.
                 target = self.get_target(offset)
                 if target <= offset:
-                    op_name = 'JUMP_BACK'
-                    if (offset in self.stmts
-                        and self.code[offset+3] not in (self.opc.END_FINALLY,
-                                                          self.opc.POP_BLOCK)):
-                        if ((offset in self.linestarts and
-                            tokens[-1].kind == 'JUMP_BACK')
-                            or offset not in self.not_continue):
-                            op_name = 'CONTINUE'
+                    op_name = "JUMP_BACK"
+                    if offset in self.stmts and self.code[offset + 3] not in (
+                        self.opc.END_FINALLY,
+                        self.opc.POP_BLOCK,
+                    ):
+                        if (
+                            offset in self.linestarts and tokens[-1].kind == "JUMP_BACK"
+                        ) or offset not in self.not_continue:
+                            op_name = "CONTINUE"
                     else:
                         # FIXME: this is a hack to catch stuff like:
                         #   if x: continue
                         # the "continue" is not on a new line.
-                        if tokens[-1].kind == 'JUMP_BACK':
+                        if tokens[-1].kind == "JUMP_BACK":
                             # We need 'intern' since we have
                             # already have processed the previous
                             # token.
-                            tokens[-1].kind = intern('CONTINUE')
+                            tokens[-1].kind = intern("CONTINUE")
 
             elif op == self.opc.LOAD_GLOBAL:
                 if offset in self.load_asserts:
@@ -314,6 +346,7 @@ class Scanner26(scan.Scanner2):
             pass
 
         if show_asm in ("both", "after"):
+            print("\n# ---- tokenization:")
             for t in tokens:
                 print(t.format(line_prefix=""))
             print()